From 4d4e3d41c3af3e90bbbdd75ff2b07fed1150d2b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Wed, 27 Apr 2022 14:55:16 +0200 Subject: [PATCH 01/15] test prompts for newsco --- promptsource/templates/cc_news/templates.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 promptsource/templates/cc_news/templates.yaml diff --git a/promptsource/templates/cc_news/templates.yaml b/promptsource/templates/cc_news/templates.yaml old mode 100755 new mode 100644 From 70823d0fd6576bb2ed31ad21a3e1c6fe072276af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Wed, 27 Apr 2022 14:56:34 +0200 Subject: [PATCH 02/15] sample prompts for zero-shot mt --- .../templates/wmt14/fr-en/templates.yaml | 198 ++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 promptsource/templates/wmt14/fr-en/templates.yaml diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml new file mode 100644 index 000000000..aa85587c4 --- /dev/null +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -0,0 +1,198 @@ +dataset: wmt14 +subset: fr-en +templates: + 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template + answer_choices: null + id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb + jinja: "Translate from French into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-fr-en-source+target + reference: '' + 2033cc05-3ef7-4c93-9684-90a618390f4b: !Template + answer_choices: null + id: 2033cc05-3ef7-4c93-9684-90a618390f4b + jinja: "What is the English translation of the French sentence: {{translation[\"\ + fr\"]}} \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-fr-en-src+target + reference: '' + 280bc983-9f20-4e80-ab02-5f333df90a42: !Template + answer_choices: null + id: 280bc983-9f20-4e80-ab02-5f333df90a42 + jinja: "What is the English translation of : {{translation[\"fr\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-fr-en-target + reference: '' + 3abec84a-592d-4282-b605-406fe4a6cc33: !Template + answer_choices: null + id: 3abec84a-592d-4282-b605-406fe4a6cc33 + jinja: 'Comment dit-on {{translation["fr"]}} en anglais ? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: comment_dit_on-fr-en-target + reference: '' + 41e7647e-2b9f-4d86-8987-5abe70000362: !Template + answer_choices: null + id: 41e7647e-2b9f-4d86-8987-5abe70000362 + jinja: "Translate into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-fr-en-target + reference: '' + 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: "Given the following passage in French: {{translation[\"fr\"]}} , a good\ + \ English translation is: \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-fr-en-source+target + reference: '' + 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20: !Template + answer_choices: null + id: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20 + jinja: "Given the following passage: {{translation[\"fr\"]}} , a good English\ + \ translation is: \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-en-fr-target + reference: '' + 4933dcae-3a66-4506-a479-a1387b287232: !Template + answer_choices: null + id: 4933dcae-3a66-4506-a479-a1387b287232 + jinja: "What is the French translation of : {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-en-fr-target + reference: '' + 51d41a04-1d90-4bae-b6e5-be7598cdbfb0: !Template + answer_choices: null + id: 51d41a04-1d90-4bae-b6e5-be7598cdbfb0 + jinja: 'How do you say {{translation["fr"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: how_to_say-fr-en-target + reference: '' + 527d2074-74fa-47ec-802e-7373728230df: !Template + answer_choices: null + id: 527d2074-74fa-47ec-802e-7373728230df + jinja: "Translate into French: {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-en-fr-target + reference: '' + 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template + answer_choices: null + id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 + jinja: "Translate from English into French: {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-en-fr-src+starget + reference: '' + 762c0878-c8fc-43ec-839f-d5d8435a94f6: !Template + answer_choices: null + id: 762c0878-c8fc-43ec-839f-d5d8435a94f6 + jinja: "Given the following passage: {{translation[\"fr\"]}} , a good English\ + \ translation is: \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-fr-en-target + reference: '' + a3a87505-e423-4c03-9a22-a3da4ccbeae5: !Template + answer_choices: null + id: a3a87505-e423-4c03-9a22-a3da4ccbeae5 + jinja: "Given the following passage in English: {{translation[\"en\"]}} , a good\ + \ French translation is: \n||| {{translation[\"fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-en-fr-source+target + reference: '' + b38093aa-3d76-433f-ade9-cbdd7c6b76de: !Template + answer_choices: null + id: b38093aa-3d76-433f-ade9-cbdd7c6b76de + jinja: "Comment dit-on {{translation[\"en\"]}} en fran\xE7ais?\n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: comment_dit_on-en-fr-target + reference: '' + c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8: !Template + answer_choices: null + id: c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8 + jinja: 'How to you say {{translation["en"]}} in French? + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: how_to_say-en-fr-target + reference: '' + ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template + answer_choices: null + id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf + jinja: "What is the French translation of the English sentence: {{translation[\"\ + en\"]}} \n||| {{translation[\"fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-en-fr-src+starget + reference: '' From 4c5d80050212b3d40f46319b6ae3fcea81ea0faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Wed, 27 Apr 2022 16:48:29 +0200 Subject: [PATCH 03/15] one more pattern for wmt14/en-fr --- .../templates/wmt14/fr-en/templates.yaml | 62 ++++++++++++++++++- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index aa85587c4..6b24afad5 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -1,10 +1,38 @@ dataset: wmt14 subset: fr-en templates: + 08ddea55-1710-4615-bbfa-fe5803e21e43: !Template + answer_choices: null + id: 08ddea55-1710-4615-bbfa-fe5803e21e43 + jinja: 'If the French version says: {{translation["fr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-fr-en-source+target + reference: '' + 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'If the English version says: {{translation["en"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-en-fr-source+target + reference: '' 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template answer_choices: null id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb - jinja: "Translate from French into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ + jinja: "Translate French into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ en\"]}}" metadata: !TemplateMetadata choices_in_prompt: false @@ -12,7 +40,7 @@ templates: - BLEU original_task: true name: translate-fr-en-source+target - reference: '' + reference: https://arxiv.org/pdf/1910.10683.pdf 2033cc05-3ef7-4c93-9684-90a618390f4b: !Template answer_choices: null id: 2033cc05-3ef7-4c93-9684-90a618390f4b @@ -126,7 +154,7 @@ templates: 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template answer_choices: null id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 - jinja: "Translate from English into French: {{translation[\"en\"]}} \n||| {{translation[\"\ + jinja: "Translate English into French: {{translation[\"en\"]}} \n||| {{translation[\"\ fr\"]}}" metadata: !TemplateMetadata choices_in_prompt: false @@ -147,6 +175,20 @@ templates: original_task: true name: a_good_translation-fr-en-target reference: '' + 9fe6b44b-2dc6-4557-8201-14d6ea7668ff: !Template + answer_choices: null + id: 9fe6b44b-2dc6-4557-8201-14d6ea7668ff + jinja: 'If the original version says: {{translation["fr"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-fr-en-target + reference: '' a3a87505-e423-4c03-9a22-a3da4ccbeae5: !Template answer_choices: null id: a3a87505-e423-4c03-9a22-a3da4ccbeae5 @@ -184,6 +226,20 @@ templates: original_task: true name: how_to_say-en-fr-target reference: '' + c80e443a-0ba4-4c5d-be98-998e050a202d: !Template + answer_choices: null + id: c80e443a-0ba4-4c5d-be98-998e050a202d + jinja: 'If the original version says: {{translation["en"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-en-fr-target + reference: '' ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template answer_choices: null id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf From e642b5684f50b78fe9f84e6b5501c79bfba05224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Thu, 28 Apr 2022 13:23:48 +0200 Subject: [PATCH 04/15] Fix an inverted fr-en prompt for wmt14 --- promptsource/templates/wmt14/fr-en/templates.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index 6b24afad5..66bb100b2 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -105,8 +105,8 @@ templates: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20: !Template answer_choices: null id: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20 - jinja: "Given the following passage: {{translation[\"fr\"]}} , a good English\ - \ translation is: \n||| {{translation[\"en\"]}}" + jinja: "Given the following passage: {{translation[\"en\"]}} , a good French translation\ + \ is: \n||| {{translation[\"fr\"]}}" metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -250,5 +250,5 @@ templates: metrics: - BLEU original_task: true - name: gpt-3-en-fr-src+starget + name: gpt-3-en-fr-src+target reference: '' From 7272271849fbab3ed164117b00c51b376abd7378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Thu, 28 Apr 2022 22:08:37 +0200 Subject: [PATCH 05/15] Prompts for WMT14 de-en, adapted directly from fr-en --- .../templates/wmt14/de-en/templates.yaml | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 promptsource/templates/wmt14/de-en/templates.yaml diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml new file mode 100644 index 000000000..63dd98f9c --- /dev/null +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -0,0 +1,254 @@ +dataset: wmt14 +subset: de-en +templates: + 08ddea55-1710-4615-bbfa-fe5803e21e43: !Template + answer_choices: null + id: 08ddea55-1710-4615-bbfa-fe5803e21e43 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-de-en-source+target + reference: '' + 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + answer_choices: null + id: 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-en-de-source+target + reference: '' + 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template + answer_choices: null + id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb + jinja: "Translate German into English: {{translation[\"de\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 2033cc05-3ef7-4c93-9684-90a618390f4b: !Template + answer_choices: null + id: 2033cc05-3ef7-4c93-9684-90a618390f4b + jinja: "What is the English translation of the German sentence: {{translation[\"\ + fr\"]}} \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-de-en-src+target + reference: '' + 280bc983-9f20-4e80-ab02-5f333df90a42: !Template + answer_choices: null + id: 280bc983-9f20-4e80-ab02-5f333df90a42 + jinja: "What is the English translation of : {{translation[\"de\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-de-en-target + reference: '' + 3abec84a-592d-4282-b605-406fe4a6cc33: !Template + answer_choices: null + id: 3abec84a-592d-4282-b605-406fe4a6cc33 + jinja: 'Comment dit-on {{translation["de"]}} en anglais ? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: comment_dit_on-de-en-target + reference: '' + 41e7647e-2b9f-4d86-8987-5abe70000362: !Template + answer_choices: null + id: 41e7647e-2b9f-4d86-8987-5abe70000362 + jinja: "Translate into English: {{translation[\"de\"]}} \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-de-en-target + reference: '' + 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + answer_choices: null + id: 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 + jinja: "Given the following passage in German: {{translation[\"de\"]}} , a good\ + \ English translation is: \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-de-en-source+target + reference: '' + 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20: !Template + answer_choices: null + id: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20 + jinja: "Given the following passage: {{translation[\"en\"]}} , a good German translation\ + \ is: \n||| {{translation[\"de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-en-de-target + reference: '' + 4933dcae-3a66-4506-a479-a1387b287232: !Template + answer_choices: null + id: 4933dcae-3a66-4506-a479-a1387b287232 + jinja: "What is the German translation of : {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-en-de-target + reference: '' + 51d41a04-1d90-4bae-b6e5-be7598cdbfb0: !Template + answer_choices: null + id: 51d41a04-1d90-4bae-b6e5-be7598cdbfb0 + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: how_to_say-de-en-target + reference: '' + 527d2074-74fa-47ec-802e-7373728230df: !Template + answer_choices: null + id: 527d2074-74fa-47ec-802e-7373728230df + jinja: "Translate into German: {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-en-de-target + reference: '' + 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template + answer_choices: null + id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 + jinja: "Translate English into German: {{translation[\"en\"]}} \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate-en-de-src+starget + reference: '' + 762c0878-c8fc-43ec-839f-d5d8435a94f6: !Template + answer_choices: null + id: 762c0878-c8fc-43ec-839f-d5d8435a94f6 + jinja: "Given the following passage: {{translation[\"de\"]}} , a good English\ + \ translation is: \n||| {{translation[\"en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-de-en-target + reference: '' + 9fe6b44b-2dc6-4557-8201-14d6ea7668ff: !Template + answer_choices: null + id: 9fe6b44b-2dc6-4557-8201-14d6ea7668ff + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-de-en-target + reference: '' + a3a87505-e423-4c03-9a22-a3da4ccbeae5: !Template + answer_choices: null + id: a3a87505-e423-4c03-9a22-a3da4ccbeae5 + jinja: "Given the following passage in English: {{translation[\"en\"]}} , a good\ + \ German translation is: \n||| {{translation[\"de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: a_good_translation-en-de-source+target + reference: '' + b38093aa-3d76-433f-ade9-cbdd7c6b76de: !Template + answer_choices: null + id: b38093aa-3d76-433f-ade9-cbdd7c6b76de + jinja: "Comment dit-on {{translation[\"en\"]}} en allemand ?\n||| {{translation[\"\ + de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: comment_dit_on-en-de-target + reference: '' + c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8: !Template + answer_choices: null + id: c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8 + jinja: 'How to you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: how_to_say-en-de-target + reference: '' + c80e443a-0ba4-4c5d-be98-998e050a202d: !Template + answer_choices: null + id: c80e443a-0ba4-4c5d-be98-998e050a202d + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: version-en-de-target + reference: '' + ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template + answer_choices: null + id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf + jinja: "What is the German translation of the English sentence: {{translation[\"\ + en\"]}} \n||| {{translation[\"de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: gpt-3-en-de-src+target + reference: '' From f086b8120adf9fcd7fa3404a85a07bc485876286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Mon, 16 May 2022 17:55:20 +0200 Subject: [PATCH 06/15] fixed duplicate uuids, add a new set of prompts --- .../templates/wmt14/de-en/templates.yaml | 142 ++++++++++++------ .../templates/wmt14/fr-en/templates.yaml | 50 +++++- 2 files changed, 144 insertions(+), 48 deletions(-) diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml index 63dd98f9c..19793d174 100644 --- a/promptsource/templates/wmt14/de-en/templates.yaml +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt14 subset: de-en templates: - 08ddea55-1710-4615-bbfa-fe5803e21e43: !Template + 40c289a0-ab68-47ed-98b6-962991d81d77: !Template answer_choices: null - id: 08ddea55-1710-4615-bbfa-fe5803e21e43 + id: 40c289a0-ab68-47ed-98b6-962991d81d77 jinja: 'If the German version says: {{translation["de"]}}; then the English version should say: @@ -15,9 +15,9 @@ templates: original_task: true name: version-de-en-source+target reference: '' - 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02: !Template + 520cc05d-6a92-4d9a-8b88-d2973ce57140: !Template answer_choices: null - id: 0bc0e46c-d1fe-4bc9-99d1-9b61aa42cd02 + id: 520cc05d-6a92-4d9a-8b88-d2973ce57140 jinja: 'If the English version says: {{translation["en"]}}; then the German version should say: @@ -29,9 +29,9 @@ templates: original_task: true name: version-en-de-source+target reference: '' - 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template + 279b2797-4137-415c-a904-76f46dd07436: !Template answer_choices: null - id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb + id: 279b2797-4137-415c-a904-76f46dd07436 jinja: "Translate German into English: {{translation[\"de\"]}} \n||| {{translation[\"\ en\"]}}" metadata: !TemplateMetadata @@ -41,11 +41,11 @@ templates: original_task: true name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - 2033cc05-3ef7-4c93-9684-90a618390f4b: !Template + 046ebd51-2862-4ad6-ae73-0af93e20a0f6: !Template answer_choices: null - id: 2033cc05-3ef7-4c93-9684-90a618390f4b + id: 046ebd51-2862-4ad6-ae73-0af93e20a0f6 jinja: "What is the English translation of the German sentence: {{translation[\"\ - fr\"]}} \n||| {{translation[\"en\"]}}" + de\"]}} \n||| {{translation[\"en\"]}}" metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -53,9 +53,9 @@ templates: original_task: true name: gpt-3-de-en-src+target reference: '' - 280bc983-9f20-4e80-ab02-5f333df90a42: !Template + 1df9f678-74bd-4a68-9c8e-e92809de2480: !Template answer_choices: null - id: 280bc983-9f20-4e80-ab02-5f333df90a42 + id: 1df9f678-74bd-4a68-9c8e-e92809de2480 jinja: "What is the English translation of : {{translation[\"de\"]}} \n||| {{translation[\"\ en\"]}}" metadata: !TemplateMetadata @@ -65,9 +65,9 @@ templates: original_task: true name: gpt-3-de-en-target reference: '' - 3abec84a-592d-4282-b605-406fe4a6cc33: !Template + b1279769-34fa-48bc-ae62-746ab3b8802d: !Template answer_choices: null - id: 3abec84a-592d-4282-b605-406fe4a6cc33 + id: b1279769-34fa-48bc-ae62-746ab3b8802d jinja: 'Comment dit-on {{translation["de"]}} en anglais ? ||| {{translation["en"]}}' @@ -78,9 +78,9 @@ templates: original_task: true name: comment_dit_on-de-en-target reference: '' - 41e7647e-2b9f-4d86-8987-5abe70000362: !Template + 21c60383-2f06-4b51-8008-65917b1f4efb: !Template answer_choices: null - id: 41e7647e-2b9f-4d86-8987-5abe70000362 + id: 21c60383-2f06-4b51-8008-65917b1f4efb jinja: "Translate into English: {{translation[\"de\"]}} \n||| {{translation[\"\ en\"]}}" metadata: !TemplateMetadata @@ -90,9 +90,9 @@ templates: original_task: true name: translate-de-en-target reference: '' - 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template + 989d6622-3e78-4074-b8ab-df0b2714f91a: !Template answer_choices: null - id: 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 + id: 989d6622-3e78-4074-b8ab-df0b2714f91a jinja: "Given the following passage in German: {{translation[\"de\"]}} , a good\ \ English translation is: \n||| {{translation[\"en\"]}}" metadata: !TemplateMetadata @@ -102,9 +102,9 @@ templates: original_task: true name: a_good_translation-de-en-source+target reference: '' - 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20: !Template + f9f86e70-550e-421d-bc2b-83042c626683: !Template answer_choices: null - id: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20 + id: f9f86e70-550e-421d-bc2b-83042c626683 jinja: "Given the following passage: {{translation[\"en\"]}} , a good German translation\ \ is: \n||| {{translation[\"de\"]}}" metadata: !TemplateMetadata @@ -114,11 +114,11 @@ templates: original_task: true name: a_good_translation-en-de-target reference: '' - 4933dcae-3a66-4506-a479-a1387b287232: !Template + fd4f81b4-e416-4afe-aee1-c017a620c580: !Template answer_choices: null - id: 4933dcae-3a66-4506-a479-a1387b287232 + id: fd4f81b4-e416-4afe-aee1-c017a620c580 jinja: "What is the German translation of : {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + de\"]}}" metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -126,9 +126,9 @@ templates: original_task: true name: gpt-3-en-de-target reference: '' - 51d41a04-1d90-4bae-b6e5-be7598cdbfb0: !Template + 7cbc8111-86d4-42a7-84fe-91ff296ab2a0: !Template answer_choices: null - id: 51d41a04-1d90-4bae-b6e5-be7598cdbfb0 + id: 7cbc8111-86d4-42a7-84fe-91ff296ab2a0 jinja: 'How do you say {{translation["de"]}} in English? ||| {{translation["en"]}}' @@ -139,11 +139,11 @@ templates: original_task: true name: how_to_say-de-en-target reference: '' - 527d2074-74fa-47ec-802e-7373728230df: !Template + 4969cca1-cabc-463a-8bab-b7556297330c: !Template answer_choices: null - id: 527d2074-74fa-47ec-802e-7373728230df + id: 4969cca1-cabc-463a-8bab-b7556297330c jinja: "Translate into German: {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + de\"]}}" metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -151,11 +151,11 @@ templates: original_task: true name: translate-en-de-target reference: '' - 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template + 147eca14-09e2-4c82-b2d8-b487bd389864: !Template answer_choices: null - id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 + id: 147eca14-09e2-4c82-b2d8-b487bd389864 jinja: "Translate English into German: {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + de\"]}}" metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -163,9 +163,9 @@ templates: original_task: true name: translate-en-de-src+starget reference: '' - 762c0878-c8fc-43ec-839f-d5d8435a94f6: !Template + c22ecc56-7a11-466c-b1f5-ebfd78176693: !Template answer_choices: null - id: 762c0878-c8fc-43ec-839f-d5d8435a94f6 + id: c22ecc56-7a11-466c-b1f5-ebfd78176693 jinja: "Given the following passage: {{translation[\"de\"]}} , a good English\ \ translation is: \n||| {{translation[\"en\"]}}" metadata: !TemplateMetadata @@ -175,9 +175,9 @@ templates: original_task: true name: a_good_translation-de-en-target reference: '' - 9fe6b44b-2dc6-4557-8201-14d6ea7668ff: !Template + 6feac2dd-5fee-4364-a420-950d45f82f4b: !Template answer_choices: null - id: 9fe6b44b-2dc6-4557-8201-14d6ea7668ff + id: 6feac2dd-5fee-4364-a420-950d45f82f4b jinja: 'If the original version says: {{translation["de"]}}; then the English version should say: @@ -189,9 +189,9 @@ templates: original_task: true name: version-de-en-target reference: '' - a3a87505-e423-4c03-9a22-a3da4ccbeae5: !Template + 145bdf7f-4f72-408d-b6ce-ce579329b345: !Template answer_choices: null - id: a3a87505-e423-4c03-9a22-a3da4ccbeae5 + id: 145bdf7f-4f72-408d-b6ce-ce579329b345 jinja: "Given the following passage in English: {{translation[\"en\"]}} , a good\ \ German translation is: \n||| {{translation[\"de\"]}}" metadata: !TemplateMetadata @@ -201,11 +201,12 @@ templates: original_task: true name: a_good_translation-en-de-source+target reference: '' - b38093aa-3d76-433f-ade9-cbdd7c6b76de: !Template + 72dff93c-73f0-4ea8-b533-4b1dc9344fc6: !Template answer_choices: null - id: b38093aa-3d76-433f-ade9-cbdd7c6b76de - jinja: "Comment dit-on {{translation[\"en\"]}} en allemand ?\n||| {{translation[\"\ - de\"]}}" + id: 72dff93c-73f0-4ea8-b533-4b1dc9344fc6 + jinja: 'Comment dit-on {{translation["en"]}} en allemand ? + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -213,10 +214,10 @@ templates: original_task: true name: comment_dit_on-en-de-target reference: '' - c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8: !Template + eb8db132-2bcb-4ccd-bac6-a19edb77a404: !Template answer_choices: null - id: c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8 - jinja: 'How to you say {{translation["en"]}} in German? + id: eb8db132-2bcb-4ccd-bac6-a19edb77a404 + jinja: 'How do you say {{translation["en"]}} in German? ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -226,9 +227,9 @@ templates: original_task: true name: how_to_say-en-de-target reference: '' - c80e443a-0ba4-4c5d-be98-998e050a202d: !Template + 85321af2-740f-4690-9def-63a308b61791: !Template answer_choices: null - id: c80e443a-0ba4-4c5d-be98-998e050a202d + id: 85321af2-740f-4690-9def-63a308b61791 jinja: 'If the original version says: {{translation["en"]}}; then the German version should say: @@ -240,9 +241,9 @@ templates: original_task: true name: version-en-de-target reference: '' - ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template + fc7ac5cb-0f08-4dc2-96f4-2738fbc71145: !Template answer_choices: null - id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf + id: fc7ac5cb-0f08-4dc2-96f4-2738fbc71145 jinja: "What is the German translation of the English sentence: {{translation[\"\ en\"]}} \n||| {{translation[\"de\"]}}" metadata: !TemplateMetadata @@ -252,3 +253,50 @@ templates: original_task: true name: gpt-3-en-de-src+target reference: '' + 6720d386-143c-4ccd-85cc-54ebdda2c280: !Template + answer_choices: null + id: 6720d386-143c-4ccd-85cc-54ebdda2c280 + jinja: "{{translation[\"de\"]}} translates into English as: \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_de-en-target + reference: '' + 250c94f6-f1dd-42f7-94b8-84666325af64: !Template + answer_choices: null + id: 250c94f6-f1dd-42f7-94b8-84666325af64 + jinja: "German: {{translation[\"de\"]}} translates into English as: \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_de-en-source+target + reference: '' + 21f54d22-0aa8-4e2f-b87f-6c595254e39e: !Template + answer_choices: null + id: 21f54d22-0aa8-4e2f-b87f-6c595254e39e + jinja: "English: {{translation[\"en\"]}} translates into German as: \n||| {{translation[\"\ + de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_en-de-source+target + reference: '' + 6259f5f2-894f-460d-a372-ea4db5a41486: !Template + answer_choices: null + id: 6259f5f2-894f-460d-a372-ea4db5a41486 + jinja: " {{translation[\"en\"]}} translates into German as: \n||| {{translation[\"\ + de\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_en-de-target diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index 66bb100b2..133d351e7 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -201,6 +201,30 @@ templates: original_task: true name: a_good_translation-en-fr-source+target reference: '' + a7fc5e89-884d-4bc5-ba8a-b500dfbd3453: !Template + answer_choices: null + id: a7fc5e89-884d-4bc5-ba8a-b500dfbd3453 + jinja: " {{translation[\"en\"]}} translates into French as: \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_en-fr-target + reference: '' + aad799b1-defb-44a9-bd51-1abcaa29fad7: !Template + answer_choices: null + id: aad799b1-defb-44a9-bd51-1abcaa29fad7 + jinja: "English: {{translation[\"en\"]}} translates into French as: \n||| {{translation[\"\ + fr\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_en-fr-source+target + reference: '' b38093aa-3d76-433f-ade9-cbdd7c6b76de: !Template answer_choices: null id: b38093aa-3d76-433f-ade9-cbdd7c6b76de @@ -213,10 +237,22 @@ templates: original_task: true name: comment_dit_on-en-fr-target reference: '' + bc95c647-5979-4522-9e6e-f5eb93b69a00: !Template + answer_choices: null + id: bc95c647-5979-4522-9e6e-f5eb93b69a00 + jinja: "French: {{translation[\"fr\"]}} translates into English as: \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_fr-en-source+target + reference: '' c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8: !Template answer_choices: null id: c50ca84e-3ca7-4c3c-89e1-3c95351f6ed8 - jinja: 'How to you say {{translation["en"]}} in French? + jinja: 'How do you say {{translation["en"]}} in French? ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -240,6 +276,18 @@ templates: original_task: true name: version-en-fr-target reference: '' + dd003b93-943e-41c9-8454-23f647e77dcc: !Template + answer_choices: null + id: dd003b93-943e-41c9-8454-23f647e77dcc + jinja: "{{translation[\"fr\"]}} translates into English as: \n||| {{translation[\"\ + en\"]}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: translate_as_fr-en-target + reference: '' ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template answer_choices: null id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf From fc14336047406b2a8a484c0dc57dc772f6c18adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Tue, 17 May 2022 17:21:09 +0200 Subject: [PATCH 07/15] restore the original template --- promptsource/templates/cc_news/templates.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 promptsource/templates/cc_news/templates.yaml diff --git a/promptsource/templates/cc_news/templates.yaml b/promptsource/templates/cc_news/templates.yaml old mode 100644 new mode 100755 From 541cc2c392250c1d3cad65d78b47bc0c3dcbda7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Tue, 24 May 2022 18:13:10 +0200 Subject: [PATCH 08/15] remove prompts in the French language to simplify copies --- .../templates/wmt14/fr-en/templates.yaml | 105 ++++++++---------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index 133d351e7..48fb73e19 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -32,8 +32,9 @@ templates: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template answer_choices: null id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb - jinja: "Translate French into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ - en\"]}}" + jinja: 'Translate French into English: {{translation["fr"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -44,8 +45,9 @@ templates: 2033cc05-3ef7-4c93-9684-90a618390f4b: !Template answer_choices: null id: 2033cc05-3ef7-4c93-9684-90a618390f4b - jinja: "What is the English translation of the French sentence: {{translation[\"\ - fr\"]}} \n||| {{translation[\"en\"]}}" + jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -56,33 +58,22 @@ templates: 280bc983-9f20-4e80-ab02-5f333df90a42: !Template answer_choices: null id: 280bc983-9f20-4e80-ab02-5f333df90a42 - jinja: "What is the English translation of : {{translation[\"fr\"]}} \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: gpt-3-fr-en-target - reference: '' - 3abec84a-592d-4282-b605-406fe4a6cc33: !Template - answer_choices: null - id: 3abec84a-592d-4282-b605-406fe4a6cc33 - jinja: 'Comment dit-on {{translation["fr"]}} en anglais ? + jinja: 'What is the English translation of : {{translation["fr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true - name: comment_dit_on-fr-en-target + name: gpt-3-fr-en-target reference: '' 41e7647e-2b9f-4d86-8987-5abe70000362: !Template answer_choices: null id: 41e7647e-2b9f-4d86-8987-5abe70000362 - jinja: "Translate into English: {{translation[\"fr\"]}} \n||| {{translation[\"\ - en\"]}}" + jinja: 'Translate into English: {{translation["fr"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -93,8 +84,8 @@ templates: 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0: !Template answer_choices: null id: 43dc1b77-e8ea-4dc8-8a12-0abc3b0dbba0 - jinja: "Given the following passage in French: {{translation[\"fr\"]}} , a good\ - \ English translation is: \n||| {{translation[\"en\"]}}" + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -105,8 +96,8 @@ templates: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20: !Template answer_choices: null id: 474c20a1-a2ea-4ff4-b4c8-7f9c6466ff20 - jinja: "Given the following passage: {{translation[\"en\"]}} , a good French translation\ - \ is: \n||| {{translation[\"fr\"]}}" + jinja: 'Given the following passage: {{translation["en"]}} , a good French translation + is: ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -117,8 +108,9 @@ templates: 4933dcae-3a66-4506-a479-a1387b287232: !Template answer_choices: null id: 4933dcae-3a66-4506-a479-a1387b287232 - jinja: "What is the French translation of : {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + jinja: 'What is the French translation of: {{translation["en"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -142,8 +134,9 @@ templates: 527d2074-74fa-47ec-802e-7373728230df: !Template answer_choices: null id: 527d2074-74fa-47ec-802e-7373728230df - jinja: "Translate into French: {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + jinja: 'Translate into French: {{translation["en"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -154,8 +147,9 @@ templates: 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template answer_choices: null id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 - jinja: "Translate English into French: {{translation[\"en\"]}} \n||| {{translation[\"\ - fr\"]}}" + jinja: 'Translate English into French: {{translation["en"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -166,8 +160,10 @@ templates: 762c0878-c8fc-43ec-839f-d5d8435a94f6: !Template answer_choices: null id: 762c0878-c8fc-43ec-839f-d5d8435a94f6 - jinja: "Given the following passage: {{translation[\"fr\"]}} , a good English\ - \ translation is: \n||| {{translation[\"en\"]}}" + jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -192,8 +188,10 @@ templates: a3a87505-e423-4c03-9a22-a3da4ccbeae5: !Template answer_choices: null id: a3a87505-e423-4c03-9a22-a3da4ccbeae5 - jinja: "Given the following passage in English: {{translation[\"en\"]}} , a good\ - \ French translation is: \n||| {{translation[\"fr\"]}}" + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good French translation is: + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -204,8 +202,9 @@ templates: a7fc5e89-884d-4bc5-ba8a-b500dfbd3453: !Template answer_choices: null id: a7fc5e89-884d-4bc5-ba8a-b500dfbd3453 - jinja: " {{translation[\"en\"]}} translates into French as: \n||| {{translation[\"\ - fr\"]}}" + jinja: ' {{translation["en"]}} translates into French as: + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -216,8 +215,9 @@ templates: aad799b1-defb-44a9-bd51-1abcaa29fad7: !Template answer_choices: null id: aad799b1-defb-44a9-bd51-1abcaa29fad7 - jinja: "English: {{translation[\"en\"]}} translates into French as: \n||| {{translation[\"\ - fr\"]}}" + jinja: 'English: {{translation["en"]}} translates into French as: + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -225,23 +225,12 @@ templates: original_task: true name: translate_as_en-fr-source+target reference: '' - b38093aa-3d76-433f-ade9-cbdd7c6b76de: !Template - answer_choices: null - id: b38093aa-3d76-433f-ade9-cbdd7c6b76de - jinja: "Comment dit-on {{translation[\"en\"]}} en fran\xE7ais?\n||| {{translation[\"\ - fr\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: comment_dit_on-en-fr-target - reference: '' bc95c647-5979-4522-9e6e-f5eb93b69a00: !Template answer_choices: null id: bc95c647-5979-4522-9e6e-f5eb93b69a00 - jinja: "French: {{translation[\"fr\"]}} translates into English as: \n||| {{translation[\"\ - en\"]}}" + jinja: 'French: {{translation["fr"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -279,8 +268,9 @@ templates: dd003b93-943e-41c9-8454-23f647e77dcc: !Template answer_choices: null id: dd003b93-943e-41c9-8454-23f647e77dcc - jinja: "{{translation[\"fr\"]}} translates into English as: \n||| {{translation[\"\ - en\"]}}" + jinja: '{{translation["fr"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -291,8 +281,9 @@ templates: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf: !Template answer_choices: null id: ec3b96b0-de5e-4ff4-b7bb-cda348ff7fcf - jinja: "What is the French translation of the English sentence: {{translation[\"\ - en\"]}} \n||| {{translation[\"fr\"]}}" + jinja: 'What is the French translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: From ba824803c31601d8c6024d8fc40f3c026ab93c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Tue, 24 May 2022 18:23:49 +0200 Subject: [PATCH 09/15] remove faulty german prompts --- .../templates/wmt14/de-en/templates.yaml | 302 ------------------ 1 file changed, 302 deletions(-) delete mode 100644 promptsource/templates/wmt14/de-en/templates.yaml diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml deleted file mode 100644 index 19793d174..000000000 --- a/promptsource/templates/wmt14/de-en/templates.yaml +++ /dev/null @@ -1,302 +0,0 @@ -dataset: wmt14 -subset: de-en -templates: - 40c289a0-ab68-47ed-98b6-962991d81d77: !Template - answer_choices: null - id: 40c289a0-ab68-47ed-98b6-962991d81d77 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: version-de-en-source+target - reference: '' - 520cc05d-6a92-4d9a-8b88-d2973ce57140: !Template - answer_choices: null - id: 520cc05d-6a92-4d9a-8b88-d2973ce57140 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: version-en-de-source+target - reference: '' - 279b2797-4137-415c-a904-76f46dd07436: !Template - answer_choices: null - id: 279b2797-4137-415c-a904-76f46dd07436 - jinja: "Translate German into English: {{translation[\"de\"]}} \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 046ebd51-2862-4ad6-ae73-0af93e20a0f6: !Template - answer_choices: null - id: 046ebd51-2862-4ad6-ae73-0af93e20a0f6 - jinja: "What is the English translation of the German sentence: {{translation[\"\ - de\"]}} \n||| {{translation[\"en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: gpt-3-de-en-src+target - reference: '' - 1df9f678-74bd-4a68-9c8e-e92809de2480: !Template - answer_choices: null - id: 1df9f678-74bd-4a68-9c8e-e92809de2480 - jinja: "What is the English translation of : {{translation[\"de\"]}} \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: gpt-3-de-en-target - reference: '' - b1279769-34fa-48bc-ae62-746ab3b8802d: !Template - answer_choices: null - id: b1279769-34fa-48bc-ae62-746ab3b8802d - jinja: 'Comment dit-on {{translation["de"]}} en anglais ? - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: comment_dit_on-de-en-target - reference: '' - 21c60383-2f06-4b51-8008-65917b1f4efb: !Template - answer_choices: null - id: 21c60383-2f06-4b51-8008-65917b1f4efb - jinja: "Translate into English: {{translation[\"de\"]}} \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate-de-en-target - reference: '' - 989d6622-3e78-4074-b8ab-df0b2714f91a: !Template - answer_choices: null - id: 989d6622-3e78-4074-b8ab-df0b2714f91a - jinja: "Given the following passage in German: {{translation[\"de\"]}} , a good\ - \ English translation is: \n||| {{translation[\"en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: a_good_translation-de-en-source+target - reference: '' - f9f86e70-550e-421d-bc2b-83042c626683: !Template - answer_choices: null - id: f9f86e70-550e-421d-bc2b-83042c626683 - jinja: "Given the following passage: {{translation[\"en\"]}} , a good German translation\ - \ is: \n||| {{translation[\"de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: a_good_translation-en-de-target - reference: '' - fd4f81b4-e416-4afe-aee1-c017a620c580: !Template - answer_choices: null - id: fd4f81b4-e416-4afe-aee1-c017a620c580 - jinja: "What is the German translation of : {{translation[\"en\"]}} \n||| {{translation[\"\ - de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: gpt-3-en-de-target - reference: '' - 7cbc8111-86d4-42a7-84fe-91ff296ab2a0: !Template - answer_choices: null - id: 7cbc8111-86d4-42a7-84fe-91ff296ab2a0 - jinja: 'How do you say {{translation["de"]}} in English? - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: how_to_say-de-en-target - reference: '' - 4969cca1-cabc-463a-8bab-b7556297330c: !Template - answer_choices: null - id: 4969cca1-cabc-463a-8bab-b7556297330c - jinja: "Translate into German: {{translation[\"en\"]}} \n||| {{translation[\"\ - de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate-en-de-target - reference: '' - 147eca14-09e2-4c82-b2d8-b487bd389864: !Template - answer_choices: null - id: 147eca14-09e2-4c82-b2d8-b487bd389864 - jinja: "Translate English into German: {{translation[\"en\"]}} \n||| {{translation[\"\ - de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate-en-de-src+starget - reference: '' - c22ecc56-7a11-466c-b1f5-ebfd78176693: !Template - answer_choices: null - id: c22ecc56-7a11-466c-b1f5-ebfd78176693 - jinja: "Given the following passage: {{translation[\"de\"]}} , a good English\ - \ translation is: \n||| {{translation[\"en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: a_good_translation-de-en-target - reference: '' - 6feac2dd-5fee-4364-a420-950d45f82f4b: !Template - answer_choices: null - id: 6feac2dd-5fee-4364-a420-950d45f82f4b - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: version-de-en-target - reference: '' - 145bdf7f-4f72-408d-b6ce-ce579329b345: !Template - answer_choices: null - id: 145bdf7f-4f72-408d-b6ce-ce579329b345 - jinja: "Given the following passage in English: {{translation[\"en\"]}} , a good\ - \ German translation is: \n||| {{translation[\"de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: a_good_translation-en-de-source+target - reference: '' - 72dff93c-73f0-4ea8-b533-4b1dc9344fc6: !Template - answer_choices: null - id: 72dff93c-73f0-4ea8-b533-4b1dc9344fc6 - jinja: 'Comment dit-on {{translation["en"]}} en allemand ? - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: comment_dit_on-en-de-target - reference: '' - eb8db132-2bcb-4ccd-bac6-a19edb77a404: !Template - answer_choices: null - id: eb8db132-2bcb-4ccd-bac6-a19edb77a404 - jinja: 'How do you say {{translation["en"]}} in German? - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: how_to_say-en-de-target - reference: '' - 85321af2-740f-4690-9def-63a308b61791: !Template - answer_choices: null - id: 85321af2-740f-4690-9def-63a308b61791 - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: version-en-de-target - reference: '' - fc7ac5cb-0f08-4dc2-96f4-2738fbc71145: !Template - answer_choices: null - id: fc7ac5cb-0f08-4dc2-96f4-2738fbc71145 - jinja: "What is the German translation of the English sentence: {{translation[\"\ - en\"]}} \n||| {{translation[\"de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: gpt-3-en-de-src+target - reference: '' - 6720d386-143c-4ccd-85cc-54ebdda2c280: !Template - answer_choices: null - id: 6720d386-143c-4ccd-85cc-54ebdda2c280 - jinja: "{{translation[\"de\"]}} translates into English as: \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate_as_de-en-target - reference: '' - 250c94f6-f1dd-42f7-94b8-84666325af64: !Template - answer_choices: null - id: 250c94f6-f1dd-42f7-94b8-84666325af64 - jinja: "German: {{translation[\"de\"]}} translates into English as: \n||| {{translation[\"\ - en\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate_as_de-en-source+target - reference: '' - 21f54d22-0aa8-4e2f-b87f-6c595254e39e: !Template - answer_choices: null - id: 21f54d22-0aa8-4e2f-b87f-6c595254e39e - jinja: "English: {{translation[\"en\"]}} translates into German as: \n||| {{translation[\"\ - de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate_as_en-de-source+target - reference: '' - 6259f5f2-894f-460d-a372-ea4db5a41486: !Template - answer_choices: null - id: 6259f5f2-894f-460d-a372-ea4db5a41486 - jinja: " {{translation[\"en\"]}} translates into German as: \n||| {{translation[\"\ - de\"]}}" - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: translate_as_en-de-target From b6a4fa9012c622f8cca041e35b2028886d8fc676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Wed, 25 May 2022 18:58:03 +0200 Subject: [PATCH 10/15] Add prompts for WMT14 hi-En --- .../templates/wmt14/hi-en/templates.yaml | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 promptsource/templates/wmt14/hi-en/templates.yaml diff --git a/promptsource/templates/wmt14/hi-en/templates.yaml b/promptsource/templates/wmt14/hi-en/templates.yaml new file mode 100644 index 000000000..c33cf783a --- /dev/null +++ b/promptsource/templates/wmt14/hi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt14 +subset: hi-en +templates: + 25a240a7-6de9-4489-a4fd-baf6660936c7: !Template + answer_choices: null + id: 25a240a7-6de9-4489-a4fd-baf6660936c7 + jinja: 'Translate Hindi into English: {{translation["hi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-hi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 31ef5fd9-a307-4c13-bd76-ac27fb6dc93f: !Template + answer_choices: null + id: 31ef5fd9-a307-4c13-bd76-ac27fb6dc93f + jinja: ' {{translation["en"]}} translates into Hindi as: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-hi-target + reference: '' + 3ab54e14-30fd-452e-84e2-76b0503a37a1: !Template + answer_choices: null + id: 3ab54e14-30fd-452e-84e2-76b0503a37a1 + jinja: 'Translate into Hindi: {{translation["en"]}} + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-hi-target + reference: '' + 3c3c0a1f-e0f0-4580-9a7c-c6911510dce0: !Template + answer_choices: null + id: 3c3c0a1f-e0f0-4580-9a7c-c6911510dce0 + jinja: 'Translate English into Hindi: {{translation["en"]}} + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-hi-src+starget + reference: '' + 4278f720-6fc5-400e-8c96-9999156506d0: !Template + answer_choices: null + id: 4278f720-6fc5-400e-8c96-9999156506d0 + jinja: 'How do you say {{translation["en"]}} in Hindi? + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-hi-target + reference: '' + 4d7d835f-188e-40cb-b6df-24f8332828a5: !Template + answer_choices: null + id: 4d7d835f-188e-40cb-b6df-24f8332828a5 + jinja: 'Given the following passage: {{translation["en"]}} , a good Hindi translation + is: ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-hi-target + reference: '' + 54360371-c0c7-453a-b5bb-f4976ba885da: !Template + answer_choices: null + id: 54360371-c0c7-453a-b5bb-f4976ba885da + jinja: 'Translate into English: {{translation["hi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-hi-en-target + reference: '' + 5e937fd7-6acb-4a5f-a72a-6e64f986f521: !Template + answer_choices: null + id: 5e937fd7-6acb-4a5f-a72a-6e64f986f521 + jinja: 'What is the Hindi translation of: {{translation["en"]}} + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-hi-target + reference: '' + 5f066596-f2cf-4077-bf44-e591642c8d1f: !Template + answer_choices: null + id: 5f066596-f2cf-4077-bf44-e591642c8d1f + jinja: 'If the Hindi version says: {{translation["hi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-hi-en-source+target + reference: '' + 6ba3186d-4494-495a-aaa1-481fb833d70a: !Template + answer_choices: null + id: 6ba3186d-4494-495a-aaa1-481fb833d70a + jinja: 'What is the Hindi translation of the English sentence: {{translation["en"]}} + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-hi-src+target + reference: '' + 708c4c05-b17b-4cd7-b27d-33de238c00f0: !Template + answer_choices: null + id: 708c4c05-b17b-4cd7-b27d-33de238c00f0 + jinja: 'If the original version says: {{translation["en"]}}; then the Hindi version + should say: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-hi-target + reference: '' + 83cd298c-4b6a-4362-909d-81a10e83ed0a: !Template + answer_choices: null + id: 83cd298c-4b6a-4362-909d-81a10e83ed0a + jinja: 'What is the English translation of the Hindi sentence: {{translation["hi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-hi-en-src+target + reference: '' + 83fa1f01-86cf-432a-b9c8-bcb876e2613c: !Template + answer_choices: null + id: 83fa1f01-86cf-432a-b9c8-bcb876e2613c + jinja: '{{translation["hi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_hi-en-target + reference: '' + 8808a912-237e-43ec-94f2-550c0a234f0f: !Template + answer_choices: null + id: 8808a912-237e-43ec-94f2-550c0a234f0f + jinja: 'If the original version says: {{translation["hi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-hi-en-target + reference: '' + 9acf9775-2bc2-4ecb-bace-7a3dbc6ebce7: !Template + answer_choices: null + id: 9acf9775-2bc2-4ecb-bace-7a3dbc6ebce7 + jinja: 'If the English version says: {{translation["en"]}}; then the Hindi version + should say: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-hi-source+target + reference: '' + 9d6b3e39-21d9-4c44-b1e4-7f808b37de7e: !Template + answer_choices: null + id: 9d6b3e39-21d9-4c44-b1e4-7f808b37de7e + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Hindi translation is: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-hi-source+target + reference: '' + adca374e-8d97-46c9-a6d7-544e100cbf38: !Template + answer_choices: null + id: adca374e-8d97-46c9-a6d7-544e100cbf38 + jinja: 'English: {{translation["en"]}} translates into Hindi as: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-hi-source+target + reference: '' + aee2cfda-d9c6-423c-a6a3-a189731e5acf: !Template + answer_choices: null + id: aee2cfda-d9c6-423c-a6a3-a189731e5acf + jinja: 'How do you say {{translation["hi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-hi-en-target + reference: '' + e64fb9f5-2c2d-44de-9d78-ffe8b73e2fc0: !Template + answer_choices: null + id: e64fb9f5-2c2d-44de-9d78-ffe8b73e2fc0 + jinja: 'Hindi: {{translation["hi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_hi-en-source+target + reference: '' + e7c5870c-5fab-4bdf-b2b2-18cabe12fb15: !Template + answer_choices: null + id: e7c5870c-5fab-4bdf-b2b2-18cabe12fb15 + jinja: 'Given the following passage: {{translation["hi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-hi-en-target + reference: '' + fca8e2a5-d633-4435-8fa8-a3727511619e: !Template + answer_choices: null + id: fca8e2a5-d633-4435-8fa8-a3727511619e + jinja: 'Given the following source text in Hindi: {{translation["hi"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-hi-en-source+target + reference: '' + fed488ee-6048-4e3c-869f-0c66d678c7ab: !Template + answer_choices: null + id: fed488ee-6048-4e3c-869f-0c66d678c7ab + jinja: 'What is the English translation of : {{translation["hi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-hi-en-target + reference: '' From 62bcefe18d943bf386c32890a321e41231a92716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Fri, 27 May 2022 10:22:43 +0200 Subject: [PATCH 11/15] Prompts for wmt14 cs-en --- .../templates/wmt14/cs-en/templates.yaml | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 promptsource/templates/wmt14/cs-en/templates.yaml diff --git a/promptsource/templates/wmt14/cs-en/templates.yaml b/promptsource/templates/wmt14/cs-en/templates.yaml new file mode 100644 index 000000000..93a48e5f6 --- /dev/null +++ b/promptsource/templates/wmt14/cs-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt14 +subset: cs-en +templates: + 09a0c104-71e9-406d-9ee2-e28ed6e5c9f5: !Template + answer_choices: null + id: 09a0c104-71e9-406d-9ee2-e28ed6e5c9f5 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-src+target + reference: '' + 2bb0c147-5d2c-4644-97a1-24f53e65f203: !Template + answer_choices: null + id: 2bb0c147-5d2c-4644-97a1-24f53e65f203 + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target + reference: '' + 3801802f-3fea-4b22-8952-dae4330a8a51: !Template + answer_choices: null + id: 3801802f-3fea-4b22-8952-dae4330a8a51 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: '' + 3b731ca7-79c0-4e28-87ba-ca44dd48ca66: !Template + answer_choices: null + id: 3b731ca7-79c0-4e28-87ba-ca44dd48ca66 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-source+target + reference: '' + 42e55c35-5fa3-4b02-89ea-98a418ccf166: !Template + answer_choices: null + id: 42e55c35-5fa3-4b02-89ea-98a418ccf166 + jinja: 'Translate English into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-src+starget + reference: '' + 4539ae0a-4d2f-4aba-a546-0b0e661799a5: !Template + answer_choices: null + id: 4539ae0a-4d2f-4aba-a546-0b0e661799a5 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + 4d723760-2c6d-40f9-b894-7517dd9a77bd: !Template + answer_choices: null + id: 4d723760-2c6d-40f9-b894-7517dd9a77bd + jinja: 'English: {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-source+target + reference: '' + 60e4f2bd-3d89-4d60-bc20-3ed0bd211289: !Template + answer_choices: null + id: 60e4f2bd-3d89-4d60-bc20-3ed0bd211289 + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 6f3e78f6-c332-400c-8b9b-27ef01c98ec6: !Template + answer_choices: null + id: 6f3e78f6-c332-400c-8b9b-27ef01c98ec6 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target + reference: '' + 73edc11b-9109-42a0-951f-632cf9d91160: !Template + answer_choices: null + id: 73edc11b-9109-42a0-951f-632cf9d91160 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + 774b3b47-c85b-4fff-b47e-3fc6bb6dc223: !Template + answer_choices: null + id: 774b3b47-c85b-4fff-b47e-3fc6bb6dc223 + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' + 7ea6d7d5-cc1d-4136-857d-00b1b5e37f81: !Template + answer_choices: null + id: 7ea6d7d5-cc1d-4136-857d-00b1b5e37f81 + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-cs-target + reference: '' + 81fe53c1-08a7-4938-b83b-7ff6d43206d5: !Template + answer_choices: null + id: 81fe53c1-08a7-4938-b83b-7ff6d43206d5 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-target + reference: '' + a0b676ea-2118-44f0-b432-c9789f6556bb: !Template + answer_choices: null + id: a0b676ea-2118-44f0-b432-c9789f6556bb + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target + reference: '' + ae09df50-7e04-4dba-bad1-251f74b654c1: !Template + answer_choices: null + id: ae09df50-7e04-4dba-bad1-251f74b654c1 + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-target + reference: '' + b210b8e8-f476-42a7-888c-e0abbbe62eca: !Template + answer_choices: null + id: b210b8e8-f476-42a7-888c-e0abbbe62eca + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + bcf3a7d3-22ac-4f4c-ac9b-7f228103ff68: !Template + answer_choices: null + id: bcf3a7d3-22ac-4f4c-ac9b-7f228103ff68 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target + reference: '' + caf37bc6-877c-495d-b7eb-591ab5aa578d: !Template + answer_choices: null + id: caf37bc6-877c-495d-b7eb-591ab5aa578d + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target + reference: '' + cc29a367-412c-4e1c-9b6f-f23703c35858: !Template + answer_choices: null + id: cc29a367-412c-4e1c-9b6f-f23703c35858 + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-target + reference: '' + e76eb1ad-7fe5-4af6-9dd3-4d85cf158c04: !Template + answer_choices: null + id: e76eb1ad-7fe5-4af6-9dd3-4d85cf158c04 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-source+target + reference: '' + f0471cf5-7531-434c-b174-fedac84ae483: !Template + answer_choices: null + id: f0471cf5-7531-434c-b174-fedac84ae483 + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target + reference: '' + fbf9be56-438b-42bd-896c-6adbebe11299: !Template + answer_choices: null + id: fbf9be56-438b-42bd-896c-6adbebe11299 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-src+target + reference: '' From dd1f445a8ce39559809ab8009b64896d66537f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Fri, 27 May 2022 18:35:00 +0200 Subject: [PATCH 12/15] prompts for wmt14-wmt19 all news tasks --- .../templates/wmt14/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt14/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt15/cs-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt15/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt15/fi-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt15/fr-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt15/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/cs-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/fi-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/ro-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt16/tr-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/cs-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/fi-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/lv-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/tr-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt17/zh-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/cs-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/et-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/fi-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/tr-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt18/zh-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/de-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/fi-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/fr-de/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/gu-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/kk-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/lt-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/ru-en/templates.yaml | 315 ++++++++++++++++++ .../templates/wmt19/zh-en/templates.yaml | 315 ++++++++++++++++++ 35 files changed, 11025 insertions(+) create mode 100644 promptsource/templates/wmt14/de-en/templates.yaml create mode 100644 promptsource/templates/wmt14/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt15/cs-en/templates.yaml create mode 100644 promptsource/templates/wmt15/de-en/templates.yaml create mode 100644 promptsource/templates/wmt15/fi-en/templates.yaml create mode 100644 promptsource/templates/wmt15/fr-en/templates.yaml create mode 100644 promptsource/templates/wmt15/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt16/cs-en/templates.yaml create mode 100644 promptsource/templates/wmt16/de-en/templates.yaml create mode 100644 promptsource/templates/wmt16/fi-en/templates.yaml create mode 100644 promptsource/templates/wmt16/ro-en/templates.yaml create mode 100644 promptsource/templates/wmt16/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt16/tr-en/templates.yaml create mode 100644 promptsource/templates/wmt17/cs-en/templates.yaml create mode 100644 promptsource/templates/wmt17/de-en/templates.yaml create mode 100644 promptsource/templates/wmt17/fi-en/templates.yaml create mode 100644 promptsource/templates/wmt17/lv-en/templates.yaml create mode 100644 promptsource/templates/wmt17/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt17/tr-en/templates.yaml create mode 100644 promptsource/templates/wmt17/zh-en/templates.yaml create mode 100644 promptsource/templates/wmt18/cs-en/templates.yaml create mode 100644 promptsource/templates/wmt18/de-en/templates.yaml create mode 100644 promptsource/templates/wmt18/et-en/templates.yaml create mode 100644 promptsource/templates/wmt18/fi-en/templates.yaml create mode 100644 promptsource/templates/wmt18/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt18/tr-en/templates.yaml create mode 100644 promptsource/templates/wmt18/zh-en/templates.yaml create mode 100644 promptsource/templates/wmt19/de-en/templates.yaml create mode 100644 promptsource/templates/wmt19/fi-en/templates.yaml create mode 100644 promptsource/templates/wmt19/fr-de/templates.yaml create mode 100644 promptsource/templates/wmt19/gu-en/templates.yaml create mode 100644 promptsource/templates/wmt19/kk-en/templates.yaml create mode 100644 promptsource/templates/wmt19/lt-en/templates.yaml create mode 100644 promptsource/templates/wmt19/ru-en/templates.yaml create mode 100644 promptsource/templates/wmt19/zh-en/templates.yaml diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml new file mode 100644 index 000000000..7fe063983 --- /dev/null +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt14 +subset: de-en +templates: + 028689e7-d0de-4bc7-902f-6a6d7b523dfe: !Template + answer_choices: null + id: 028689e7-d0de-4bc7-902f-6a6d7b523dfe + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + 2daaac1b-1431-4b18-98ba-f9e2479a79cf: !Template + answer_choices: null + id: 2daaac1b-1431-4b18-98ba-f9e2479a79cf + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 3284f6a8-b441-493e-b76c-4e9116d03eaa: !Template + answer_choices: null + id: 3284f6a8-b441-493e-b76c-4e9116d03eaa + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + 477d648c-6d23-4757-b21d-1a690e2cb61d: !Template + answer_choices: null + id: 477d648c-6d23-4757-b21d-1a690e2cb61d + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + 47b3f216-954c-4bf9-8347-cc49b9587f7f: !Template + answer_choices: null + id: 47b3f216-954c-4bf9-8347-cc49b9587f7f + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + 4876ebc0-53fe-447c-ac1e-88b8d7fdc1cb: !Template + answer_choices: null + id: 4876ebc0-53fe-447c-ac1e-88b8d7fdc1cb + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 7035f352-28ed-4025-af47-faf9e60680ba: !Template + answer_choices: null + id: 7035f352-28ed-4025-af47-faf9e60680ba + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + 74753485-14e5-4acd-a82b-6eb12a4ac3ea: !Template + answer_choices: null + id: 74753485-14e5-4acd-a82b-6eb12a4ac3ea + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + 787c693b-0759-4b1b-b9ef-3ec9ba9f2c37: !Template + answer_choices: null + id: 787c693b-0759-4b1b-b9ef-3ec9ba9f2c37 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + 7af42f5e-98e8-4248-a54e-dec21045ba0b: !Template + answer_choices: null + id: 7af42f5e-98e8-4248-a54e-dec21045ba0b + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + 8966ea5e-92ad-4fcb-8aa0-5f2207ba4acb: !Template + answer_choices: null + id: 8966ea5e-92ad-4fcb-8aa0-5f2207ba4acb + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' + 8abfaaee-565f-40b9-8a20-6731f35aa0a6: !Template + answer_choices: null + id: 8abfaaee-565f-40b9-8a20-6731f35aa0a6 + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + 99d4e661-6ec3-4f21-ac52-64a8d3e42374: !Template + answer_choices: null + id: 99d4e661-6ec3-4f21-ac52-64a8d3e42374 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + a1598a2a-c3ec-413c-9ae1-727dccec8a88: !Template + answer_choices: null + id: a1598a2a-c3ec-413c-9ae1-727dccec8a88 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + a85989e5-787e-4953-9f29-1e1b453a7413: !Template + answer_choices: null + id: a85989e5-787e-4953-9f29-1e1b453a7413 + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' + aaf593d0-4bc8-4d35-9f14-c2545ae77d6b: !Template + answer_choices: null + id: aaf593d0-4bc8-4d35-9f14-c2545ae77d6b + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + c11e1bd0-a07d-4cd5-9873-621dc5764fda: !Template + answer_choices: null + id: c11e1bd0-a07d-4cd5-9873-621dc5764fda + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + cd13a8e4-26c8-4f51-95ea-04025752ebb2: !Template + answer_choices: null + id: cd13a8e4-26c8-4f51-95ea-04025752ebb2 + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + dc9f3ef3-eab9-4fb0-b825-35ce78dcb835: !Template + answer_choices: null + id: dc9f3ef3-eab9-4fb0-b825-35ce78dcb835 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + eb2dff5a-8097-48e6-a24a-fc5f0281aa20: !Template + answer_choices: null + id: eb2dff5a-8097-48e6-a24a-fc5f0281aa20 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + f927c8f6-2cf4-47ed-8641-124912eea736: !Template + answer_choices: null + id: f927c8f6-2cf4-47ed-8641-124912eea736 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + f9bb17d8-3e15-4865-9f2e-bd23077ea77c: !Template + answer_choices: null + id: f9bb17d8-3e15-4865-9f2e-bd23077ea77c + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' diff --git a/promptsource/templates/wmt14/ru-en/templates.yaml b/promptsource/templates/wmt14/ru-en/templates.yaml new file mode 100644 index 000000000..82bc4af3e --- /dev/null +++ b/promptsource/templates/wmt14/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt14 +subset: ru-en +templates: + 0026c4d6-5b4d-440f-ba9a-df7894b4f92c: !Template + answer_choices: null + id: 0026c4d6-5b4d-440f-ba9a-df7894b4f92c + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + 2e565e0b-0beb-4a22-a603-7f4fa96123d2: !Template + answer_choices: null + id: 2e565e0b-0beb-4a22-a603-7f4fa96123d2 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' + 30e887af-c0e6-4063-9e60-4bc226cbfd3e: !Template + answer_choices: null + id: 30e887af-c0e6-4063-9e60-4bc226cbfd3e + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + 4738c193-8c27-42ca-9217-20d16bfe892c: !Template + answer_choices: null + id: 4738c193-8c27-42ca-9217-20d16bfe892c + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 53558679-249a-4205-896f-0b139fc9bd55: !Template + answer_choices: null + id: 53558679-249a-4205-896f-0b139fc9bd55 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + 539ef9e1-4d07-40a6-a2f5-80dad964e2b7: !Template + answer_choices: null + id: 539ef9e1-4d07-40a6-a2f5-80dad964e2b7 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + 57ff898b-9254-4d5d-a002-affd6f41fe9f: !Template + answer_choices: null + id: 57ff898b-9254-4d5d-a002-affd6f41fe9f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + 606cfce9-21eb-4f04-b543-563750d4c6d3: !Template + answer_choices: null + id: 606cfce9-21eb-4f04-b543-563750d4c6d3 + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + 643776a7-0520-4d5c-864d-f5c6a8342b38: !Template + answer_choices: null + id: 643776a7-0520-4d5c-864d-f5c6a8342b38 + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 65a3b705-0dcc-4283-b585-763ab682db4a: !Template + answer_choices: null + id: 65a3b705-0dcc-4283-b585-763ab682db4a + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' + 7cd72da5-9047-4712-b2f2-75d04a0b9457: !Template + answer_choices: null + id: 7cd72da5-9047-4712-b2f2-75d04a0b9457 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' + 82a01d5b-4bff-41a0-9cd4-9d76cabce590: !Template + answer_choices: null + id: 82a01d5b-4bff-41a0-9cd4-9d76cabce590 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + 8a288d1f-757e-4aa7-a7ac-66bb01c29f65: !Template + answer_choices: null + id: 8a288d1f-757e-4aa7-a7ac-66bb01c29f65 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + 8c8d404c-1c7b-4612-8764-c1c905982c17: !Template + answer_choices: null + id: 8c8d404c-1c7b-4612-8764-c1c905982c17 + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + 9564f76a-d551-4d14-89e1-0a532327a0cb: !Template + answer_choices: null + id: 9564f76a-d551-4d14-89e1-0a532327a0cb + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + a3e7bc85-f1c3-41bf-b4b3-d3dbaba820e9: !Template + answer_choices: null + id: a3e7bc85-f1c3-41bf-b4b3-d3dbaba820e9 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + b242c63d-1530-47f5-8df1-0d403fccb44b: !Template + answer_choices: null + id: b242c63d-1530-47f5-8df1-0d403fccb44b + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' + b82523d6-80c0-42fd-a39a-b8001b67b29a: !Template + answer_choices: null + id: b82523d6-80c0-42fd-a39a-b8001b67b29a + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + bfbb1dfa-5a7d-4e9c-bebb-49dc6e132eb0: !Template + answer_choices: null + id: bfbb1dfa-5a7d-4e9c-bebb-49dc6e132eb0 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + d794ed7f-d098-4802-8328-09244186cec8: !Template + answer_choices: null + id: d794ed7f-d098-4802-8328-09244186cec8 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + f568cbb6-e2a6-4cb2-b51d-cdb36e50df18: !Template + answer_choices: null + id: f568cbb6-e2a6-4cb2-b51d-cdb36e50df18 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + fe8533e6-df7f-46c4-b67e-e0657cd199d2: !Template + answer_choices: null + id: fe8533e6-df7f-46c4-b67e-e0657cd199d2 + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' diff --git a/promptsource/templates/wmt15/cs-en/templates.yaml b/promptsource/templates/wmt15/cs-en/templates.yaml new file mode 100644 index 000000000..7de8313fd --- /dev/null +++ b/promptsource/templates/wmt15/cs-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt15 +subset: cs-en +templates: + 06f0d97a-6082-43f4-b581-d59d31e5bce9: !Template + answer_choices: null + id: 06f0d97a-6082-43f4-b581-d59d31e5bce9 + jinja: 'English: {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-source+target + reference: '' + 0fbce2e7-b196-4995-a774-de4464474b54: !Template + answer_choices: null + id: 0fbce2e7-b196-4995-a774-de4464474b54 + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-cs-target + reference: '' + 1dd6f3bc-a34d-4c02-a62e-3a2c7af9f160: !Template + answer_choices: null + id: 1dd6f3bc-a34d-4c02-a62e-3a2c7af9f160 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + 2b02f0d7-6940-4503-a0b8-7fdb6473f688: !Template + answer_choices: null + id: 2b02f0d7-6940-4503-a0b8-7fdb6473f688 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-source+target + reference: '' + 31bf20ab-ac14-4da5-ae90-46028b960beb: !Template + answer_choices: null + id: 31bf20ab-ac14-4da5-ae90-46028b960beb + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target + reference: '' + 3237a771-40b9-476c-811c-36c59194524f: !Template + answer_choices: null + id: 3237a771-40b9-476c-811c-36c59194524f + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target + reference: '' + 3545b365-136e-4a8b-ab6f-1be700ee4d5e: !Template + answer_choices: null + id: 3545b365-136e-4a8b-ab6f-1be700ee4d5e + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target + reference: '' + 5f8f7d5c-7ca9-40f9-b979-382eb911ca9a: !Template + answer_choices: null + id: 5f8f7d5c-7ca9-40f9-b979-382eb911ca9a + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + 668cd91b-377c-4c36-82d3-ee7eeab17948: !Template + answer_choices: null + id: 668cd91b-377c-4c36-82d3-ee7eeab17948 + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target + reference: '' + 69eb76fd-ec6c-42ec-9459-6946f86fef81: !Template + answer_choices: null + id: 69eb76fd-ec6c-42ec-9459-6946f86fef81 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-src+target + reference: '' + 6c5e1d66-4521-436e-b609-34b1b3c714f8: !Template + answer_choices: null + id: 6c5e1d66-4521-436e-b609-34b1b3c714f8 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-source+target + reference: '' + 7d0ad92c-50e7-4be7-9557-aa5f18fd2b0f: !Template + answer_choices: null + id: 7d0ad92c-50e7-4be7-9557-aa5f18fd2b0f + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + 85ea7e61-2ee1-4993-8f9f-a870ef0bd140: !Template + answer_choices: null + id: 85ea7e61-2ee1-4993-8f9f-a870ef0bd140 + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-target + reference: '' + 976d9e4c-e7fb-4e3c-837e-a494bd7df6cb: !Template + answer_choices: null + id: 976d9e4c-e7fb-4e3c-837e-a494bd7df6cb + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + a9635589-6b1e-4735-8afb-6c8c5a6212f7: !Template + answer_choices: null + id: a9635589-6b1e-4735-8afb-6c8c5a6212f7 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-target + reference: '' + acca0487-3ff6-4572-b07f-7d740bb8c357: !Template + answer_choices: null + id: acca0487-3ff6-4572-b07f-7d740bb8c357 + jinja: 'Translate English into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-src+starget + reference: '' + b1366146-ea0d-4230-aa5d-bbbeb991977b: !Template + answer_choices: null + id: b1366146-ea0d-4230-aa5d-bbbeb991977b + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-target + reference: '' + b37f5d69-409f-491f-9c6e-0963caf0088a: !Template + answer_choices: null + id: b37f5d69-409f-491f-9c6e-0963caf0088a + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-src+target + reference: '' + cad8d76f-721d-40a4-a927-d6a4c42a7904: !Template + answer_choices: null + id: cad8d76f-721d-40a4-a927-d6a4c42a7904 + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' + e2511ab0-a61a-4502-9126-7eb3afa9f8d6: !Template + answer_choices: null + id: e2511ab0-a61a-4502-9126-7eb3afa9f8d6 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: '' + f701fee7-a397-4410-a7a9-1e3616de9313: !Template + answer_choices: null + id: f701fee7-a397-4410-a7a9-1e3616de9313 + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target + reference: '' + fbb5e375-f3f0-4701-bdcb-c4587ea39d08: !Template + answer_choices: null + id: fbb5e375-f3f0-4701-bdcb-c4587ea39d08 + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target + reference: '' diff --git a/promptsource/templates/wmt15/de-en/templates.yaml b/promptsource/templates/wmt15/de-en/templates.yaml new file mode 100644 index 000000000..7548b0192 --- /dev/null +++ b/promptsource/templates/wmt15/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt15 +subset: de-en +templates: + 01ab38fd-50c7-4670-9c02-7c7401e0418a: !Template + answer_choices: null + id: 01ab38fd-50c7-4670-9c02-7c7401e0418a + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + 05195a8f-6f76-48f8-9db1-d155e76314e5: !Template + answer_choices: null + id: 05195a8f-6f76-48f8-9db1-d155e76314e5 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + 0b6024e1-cc9d-4dda-b5ba-1c840e5dda8b: !Template + answer_choices: null + id: 0b6024e1-cc9d-4dda-b5ba-1c840e5dda8b + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' + 0e74e533-e01e-401f-8316-81fcdd11483a: !Template + answer_choices: null + id: 0e74e533-e01e-401f-8316-81fcdd11483a + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 1023ac76-49f6-48cb-a4aa-53965ffba794: !Template + answer_choices: null + id: 1023ac76-49f6-48cb-a4aa-53965ffba794 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + 140eb0aa-b6ed-45e6-bd41-fb6e6c8a9cd8: !Template + answer_choices: null + id: 140eb0aa-b6ed-45e6-bd41-fb6e6c8a9cd8 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 231e54cf-c967-4998-84cb-f5feaa2bb461: !Template + answer_choices: null + id: 231e54cf-c967-4998-84cb-f5feaa2bb461 + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' + 2c87a41f-51dd-4802-9e88-2335a31e8651: !Template + answer_choices: null + id: 2c87a41f-51dd-4802-9e88-2335a31e8651 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + 3cb3d385-f0c7-4626-8f3e-754d6efcd314: !Template + answer_choices: null + id: 3cb3d385-f0c7-4626-8f3e-754d6efcd314 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + 485aa919-0e7e-46cf-b8f5-d8b294599b01: !Template + answer_choices: null + id: 485aa919-0e7e-46cf-b8f5-d8b294599b01 + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8e11df98-cad8-4236-b3e3-7abc130863cc: !Template + answer_choices: null + id: 8e11df98-cad8-4236-b3e3-7abc130863cc + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 9af3221d-970f-429c-a4f3-7985a53e8fc3: !Template + answer_choices: null + id: 9af3221d-970f-429c-a4f3-7985a53e8fc3 + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' + 9c93367c-d0a0-4fc7-84ed-4e0ac5361416: !Template + answer_choices: null + id: 9c93367c-d0a0-4fc7-84ed-4e0ac5361416 + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + 9cf7d339-577c-4b93-8584-912158c67080: !Template + answer_choices: null + id: 9cf7d339-577c-4b93-8584-912158c67080 + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + b2180a7e-6917-4214-9189-d0b24bb850b9: !Template + answer_choices: null + id: b2180a7e-6917-4214-9189-d0b24bb850b9 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + b32d4b5d-a34e-4567-a880-43209f78edb7: !Template + answer_choices: null + id: b32d4b5d-a34e-4567-a880-43209f78edb7 + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + ca0d2e57-2fda-418c-a4b6-f2626768c103: !Template + answer_choices: null + id: ca0d2e57-2fda-418c-a4b6-f2626768c103 + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + d01c2154-db01-4689-b59f-1329d85e527b: !Template + answer_choices: null + id: d01c2154-db01-4689-b59f-1329d85e527b + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + dcfdd38d-c333-4480-9cf8-60136d2dfae2: !Template + answer_choices: null + id: dcfdd38d-c333-4480-9cf8-60136d2dfae2 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + e1108e0c-02cc-449e-995c-57603d161279: !Template + answer_choices: null + id: e1108e0c-02cc-449e-995c-57603d161279 + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + e3a7ab9a-fb95-4c93-8aba-71208450998d: !Template + answer_choices: null + id: e3a7ab9a-fb95-4c93-8aba-71208450998d + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + f0ae334c-9ea0-46bc-b474-a988218b4e6a: !Template + answer_choices: null + id: f0ae334c-9ea0-46bc-b474-a988218b4e6a + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' diff --git a/promptsource/templates/wmt15/fi-en/templates.yaml b/promptsource/templates/wmt15/fi-en/templates.yaml new file mode 100644 index 000000000..0ce712f37 --- /dev/null +++ b/promptsource/templates/wmt15/fi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt15 +subset: fi-en +templates: + 0d4cc6d9-733f-4c67-9f79-a6d27b1f858f: !Template + answer_choices: null + id: 0d4cc6d9-733f-4c67-9f79-a6d27b1f858f + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-src+target + reference: '' + 1c3f5536-818f-475a-89e4-6c62e22d439c: !Template + answer_choices: null + id: 1c3f5536-818f-475a-89e4-6c62e22d439c + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-source+target + reference: '' + 1cc12cec-8354-4025-817d-839ea1787e1a: !Template + answer_choices: null + id: 1cc12cec-8354-4025-817d-839ea1787e1a + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + 2271f128-3ae4-4704-84ee-53028f57d926: !Template + answer_choices: null + id: 2271f128-3ae4-4704-84ee-53028f57d926 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + 3cf8e8eb-c4aa-4211-8779-97a4f133546e: !Template + answer_choices: null + id: 3cf8e8eb-c4aa-4211-8779-97a4f133546e + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-target + reference: '' + 4e34023a-b5fb-428b-979d-374126074714: !Template + answer_choices: null + id: 4e34023a-b5fb-428b-979d-374126074714 + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target + reference: '' + 4fe7ff1c-d803-45da-8fb7-ada4f3c70fe2: !Template + answer_choices: null + id: 4fe7ff1c-d803-45da-8fb7-ada4f3c70fe2 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target + reference: '' + 663ee2dd-d784-4fc4-a0d5-25705c000ef0: !Template + answer_choices: null + id: 663ee2dd-d784-4fc4-a0d5-25705c000ef0 + jinja: 'How do you say {{translation["fi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fi-en-target + reference: '' + 77757cf4-6946-414a-967d-136ce9a17b50: !Template + answer_choices: null + id: 77757cf4-6946-414a-967d-136ce9a17b50 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + 7a2a2a98-662e-45ea-b1ea-cced5318feb8: !Template + answer_choices: null + id: 7a2a2a98-662e-45ea-b1ea-cced5318feb8 + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-target + reference: '' + 8bdabcfa-3d40-4d45-b7cb-1806570397fe: !Template + answer_choices: null + id: 8bdabcfa-3d40-4d45-b7cb-1806570397fe + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-src+starget + reference: '' + 95a1bcc5-fb23-4fec-b81d-93eaa955d615: !Template + answer_choices: null + id: 95a1bcc5-fb23-4fec-b81d-93eaa955d615 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-target + reference: '' + 9639fd0e-1f76-446d-9c7a-0d9c3a10dca2: !Template + answer_choices: null + id: 9639fd0e-1f76-446d-9c7a-0d9c3a10dca2 + jinja: 'Translate Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 993eb10a-38d7-4880-9432-e73834b06893: !Template + answer_choices: null + id: 993eb10a-38d7-4880-9432-e73834b06893 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + a799d0ab-ece3-405b-a492-d0f177767f1e: !Template + answer_choices: null + id: a799d0ab-ece3-405b-a492-d0f177767f1e + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: '' + aa6dba99-10d7-4fd0-a5c1-263de99e3759: !Template + answer_choices: null + id: aa6dba99-10d7-4fd0-a5c1-263de99e3759 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target + reference: '' + afe000ca-c5f9-4138-85ed-4d1c073f3d10: !Template + answer_choices: null + id: afe000ca-c5f9-4138-85ed-4d1c073f3d10 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-target + reference: '' + baba0fc5-1dad-423f-8b11-90f3f06b0bb2: !Template + answer_choices: null + id: baba0fc5-1dad-423f-8b11-90f3f06b0bb2 + jinja: 'What is the Finnish translation of: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-target + reference: '' + cf22aa75-de10-42c0-9141-4fc2953fe1b7: !Template + answer_choices: null + id: cf22aa75-de10-42c0-9141-4fc2953fe1b7 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-src+target + reference: '' + cf59bd05-215a-4dfd-9634-aa2c6c8147cc: !Template + answer_choices: null + id: cf59bd05-215a-4dfd-9634-aa2c6c8147cc + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target + reference: '' + eb22bd68-0c81-4389-9370-64f8cdd0e037: !Template + answer_choices: null + id: eb22bd68-0c81-4389-9370-64f8cdd0e037 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-source+target + reference: '' + f618b9c0-6c74-450d-bf95-d711ad40f457: !Template + answer_choices: null + id: f618b9c0-6c74-450d-bf95-d711ad40f457 + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-source+target + reference: '' diff --git a/promptsource/templates/wmt15/fr-en/templates.yaml b/promptsource/templates/wmt15/fr-en/templates.yaml new file mode 100644 index 000000000..accda865e --- /dev/null +++ b/promptsource/templates/wmt15/fr-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt15 +subset: fr-en +templates: + 086d3670-4e19-488d-a12a-3c635b0b0675: !Template + answer_choices: null + id: 086d3670-4e19-488d-a12a-3c635b0b0675 + jinja: 'Translate French into English: {{translation["fr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 0c485a59-4f19-43ea-a228-dd9daa95581a: !Template + answer_choices: null + id: 0c485a59-4f19-43ea-a228-dd9daa95581a + jinja: 'English: {{translation["en"]}} translates into French as: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fr-source+target + reference: '' + 0d317972-73b8-4f8d-9c9d-e3fcdfa39781: !Template + answer_choices: null + id: 0d317972-73b8-4f8d-9c9d-e3fcdfa39781 + jinja: 'Translate into English: {{translation["fr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fr-en-target + reference: '' + 10b1b2d8-4ade-4545-b9cd-0bdc20f83b17: !Template + answer_choices: null + id: 10b1b2d8-4ade-4545-b9cd-0bdc20f83b17 + jinja: 'Translate English into French: {{translation["en"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fr-src+starget + reference: '' + 13b8c6a0-b968-43ed-bdf5-3ca2acc98c40: !Template + answer_choices: null + id: 13b8c6a0-b968-43ed-bdf5-3ca2acc98c40 + jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fr-en-target + reference: '' + 15e9c7fb-31b3-44f2-b56c-3dac3aa921d5: !Template + answer_choices: null + id: 15e9c7fb-31b3-44f2-b56c-3dac3aa921d5 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good French translation is: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fr-source+target + reference: '' + 1bfca273-4e3b-42dc-ba5d-11a2823d3a5a: !Template + answer_choices: null + id: 1bfca273-4e3b-42dc-ba5d-11a2823d3a5a + jinja: 'Given the following passage: {{translation["en"]}} , a good French translation + is: ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fr-target + reference: '' + 226f7037-a37d-4154-a5f5-d613afc3ec76: !Template + answer_choices: null + id: 226f7037-a37d-4154-a5f5-d613afc3ec76 + jinja: 'What is the French translation of: {{translation["en"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fr-target + reference: '' + 405e991d-ec19-4a49-a45e-a2ebe98044a0: !Template + answer_choices: null + id: 405e991d-ec19-4a49-a45e-a2ebe98044a0 + jinja: 'What is the English translation of : {{translation["fr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fr-en-target + reference: '' + 40ed2bb6-4007-48bd-bf3f-569784505916: !Template + answer_choices: null + id: 40ed2bb6-4007-48bd-bf3f-569784505916 + jinja: 'If the French version says: {{translation["fr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fr-en-source+target + reference: '' + 432c00d7-74c8-415d-b931-a7cc05211a7a: !Template + answer_choices: null + id: 432c00d7-74c8-415d-b931-a7cc05211a7a + jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fr-en-src+target + reference: '' + 5b1b257b-457d-4a7e-9a2a-ef6924ba6b01: !Template + answer_choices: null + id: 5b1b257b-457d-4a7e-9a2a-ef6924ba6b01 + jinja: 'Translate into French: {{translation["en"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fr-target + reference: '' + 8ce8cd9f-8c76-4559-8a40-9b4b6efc1408: !Template + answer_choices: null + id: 8ce8cd9f-8c76-4559-8a40-9b4b6efc1408 + jinja: 'How do you say {{translation["fr"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fr-en-target + reference: '' + 8d262474-b465-45b2-b39a-0f1f95b0a9c0: !Template + answer_choices: null + id: 8d262474-b465-45b2-b39a-0f1f95b0a9c0 + jinja: 'French: {{translation["fr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fr-en-source+target + reference: '' + 9180b8ec-ec77-47cf-a930-3832691cb210: !Template + answer_choices: null + id: 9180b8ec-ec77-47cf-a930-3832691cb210 + jinja: 'If the original version says: {{translation["en"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fr-target + reference: '' + a0bc23d9-664b-45b8-9ccd-b3c16f70b28c: !Template + answer_choices: null + id: a0bc23d9-664b-45b8-9ccd-b3c16f70b28c + jinja: 'If the original version says: {{translation["fr"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fr-en-target + reference: '' + a38e45aa-12ce-4c64-a2d9-76f184aec707: !Template + answer_choices: null + id: a38e45aa-12ce-4c64-a2d9-76f184aec707 + jinja: 'How do you say {{translation["en"]}} in French? + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fr-target + reference: '' + a6d3c7fe-e155-4677-bbf6-30566f29faf9: !Template + answer_choices: null + id: a6d3c7fe-e155-4677-bbf6-30566f29faf9 + jinja: 'What is the French translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fr-src+target + reference: '' + b2b616a0-b613-4fa3-86b6-efca11803f9c: !Template + answer_choices: null + id: b2b616a0-b613-4fa3-86b6-efca11803f9c + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fr-en-source+target + reference: '' + bb9cc49c-adcd-487b-8fac-49847fa4c680: !Template + answer_choices: null + id: bb9cc49c-adcd-487b-8fac-49847fa4c680 + jinja: '{{translation["fr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fr-en-target + reference: '' + c205c9cc-f09f-4d55-ae7b-c81337ee95e1: !Template + answer_choices: null + id: c205c9cc-f09f-4d55-ae7b-c81337ee95e1 + jinja: 'If the English version says: {{translation["en"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fr-source+target + reference: '' + ff2ca060-1384-4836-b0cd-6c48c84bf9ae: !Template + answer_choices: null + id: ff2ca060-1384-4836-b0cd-6c48c84bf9ae + jinja: ' {{translation["en"]}} translates into French as: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fr-target + reference: '' diff --git a/promptsource/templates/wmt15/ru-en/templates.yaml b/promptsource/templates/wmt15/ru-en/templates.yaml new file mode 100644 index 000000000..358b58b61 --- /dev/null +++ b/promptsource/templates/wmt15/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt15 +subset: ru-en +templates: + 1318da6d-555b-4c5d-aa31-5f8b4a820a8d: !Template + answer_choices: null + id: 1318da6d-555b-4c5d-aa31-5f8b4a820a8d + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 155a3c3c-ef8e-4956-93cb-4d91231e3b9c: !Template + answer_choices: null + id: 155a3c3c-ef8e-4956-93cb-4d91231e3b9c + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' + 41902388-737e-4a3c-9bdf-efc470895d14: !Template + answer_choices: null + id: 41902388-737e-4a3c-9bdf-efc470895d14 + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + 48c610c6-72f5-4665-9822-addf0bd0037d: !Template + answer_choices: null + id: 48c610c6-72f5-4665-9822-addf0bd0037d + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + 67dcb08c-c4eb-4da7-8249-0d51c04864d3: !Template + answer_choices: null + id: 67dcb08c-c4eb-4da7-8249-0d51c04864d3 + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 6f1eeb41-0174-4d9a-a5da-942c4017d352: !Template + answer_choices: null + id: 6f1eeb41-0174-4d9a-a5da-942c4017d352 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + 6f239ff6-5f43-4951-959f-1cd39359cc60: !Template + answer_choices: null + id: 6f239ff6-5f43-4951-959f-1cd39359cc60 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + 73821062-32c8-4589-8c35-274f75ff1bc1: !Template + answer_choices: null + id: 73821062-32c8-4589-8c35-274f75ff1bc1 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' + 75d10f69-f208-4795-bc0e-032c6d5d1744: !Template + answer_choices: null + id: 75d10f69-f208-4795-bc0e-032c6d5d1744 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' + 77cc43eb-c7df-4618-9707-36bd04038787: !Template + answer_choices: null + id: 77cc43eb-c7df-4618-9707-36bd04038787 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + a2ac1716-1abb-467e-9bee-cff01bf1beab: !Template + answer_choices: null + id: a2ac1716-1abb-467e-9bee-cff01bf1beab + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + afc46ae1-90f4-4db0-a576-3f5804071f46: !Template + answer_choices: null + id: afc46ae1-90f4-4db0-a576-3f5804071f46 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + c3be2d8a-23db-46d5-9603-06cb7e6c9e80: !Template + answer_choices: null + id: c3be2d8a-23db-46d5-9603-06cb7e6c9e80 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + c9d135b5-0740-4a36-a8b3-f45a14380d32: !Template + answer_choices: null + id: c9d135b5-0740-4a36-a8b3-f45a14380d32 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + d8a8472e-d8b2-41e0-bbf8-1c906e63c6cd: !Template + answer_choices: null + id: d8a8472e-d8b2-41e0-bbf8-1c906e63c6cd + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + e0c37fa9-2597-4482-aad4-578369694cc4: !Template + answer_choices: null + id: e0c37fa9-2597-4482-aad4-578369694cc4 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + e2581600-24bd-4df3-a326-df1d285b7925: !Template + answer_choices: null + id: e2581600-24bd-4df3-a326-df1d285b7925 + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + ee837cf8-ae01-46de-8157-be27d9d14515: !Template + answer_choices: null + id: ee837cf8-ae01-46de-8157-be27d9d14515 + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' + f3bec842-663c-4300-b496-616216ceff6a: !Template + answer_choices: null + id: f3bec842-663c-4300-b496-616216ceff6a + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + f6409e35-9e64-4dea-a637-648bf33cb8ac: !Template + answer_choices: null + id: f6409e35-9e64-4dea-a637-648bf33cb8ac + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + fbe341d6-74a0-4af4-b5a4-e9c1f81ff5c7: !Template + answer_choices: null + id: fbe341d6-74a0-4af4-b5a4-e9c1f81ff5c7 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' + fd0c2f56-55b4-42e9-9867-be1af5b2584c: !Template + answer_choices: null + id: fd0c2f56-55b4-42e9-9867-be1af5b2584c + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' diff --git a/promptsource/templates/wmt16/cs-en/templates.yaml b/promptsource/templates/wmt16/cs-en/templates.yaml new file mode 100644 index 000000000..3062f18a2 --- /dev/null +++ b/promptsource/templates/wmt16/cs-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: cs-en +templates: + 0892dd27-5006-4066-b388-ddc21920ca1b: !Template + answer_choices: null + id: 0892dd27-5006-4066-b388-ddc21920ca1b + jinja: 'Translate English into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-src+starget + reference: '' + 10cb3276-9c00-471f-97ad-35ecadaff00b: !Template + answer_choices: null + id: 10cb3276-9c00-471f-97ad-35ecadaff00b + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target + reference: '' + 2098c870-299b-44ab-a7fa-44b347cb7294: !Template + answer_choices: null + id: 2098c870-299b-44ab-a7fa-44b347cb7294 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-source+target + reference: '' + 24d73443-a15a-42cc-99a3-7fac8f4bc67f: !Template + answer_choices: null + id: 24d73443-a15a-42cc-99a3-7fac8f4bc67f + jinja: 'English: {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-source+target + reference: '' + 258a4b04-aea6-4ea2-9e05-68252a71a9ab: !Template + answer_choices: null + id: 258a4b04-aea6-4ea2-9e05-68252a71a9ab + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-target + reference: '' + 25d84bda-fb1d-4239-8fc0-64f6088a1e01: !Template + answer_choices: null + id: 25d84bda-fb1d-4239-8fc0-64f6088a1e01 + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 2976ce81-af77-4f32-b74b-4a5b889660ee: !Template + answer_choices: null + id: 2976ce81-af77-4f32-b74b-4a5b889660ee + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-src+target + reference: '' + 2cccae35-0560-4b41-9ff4-07e59cef3bee: !Template + answer_choices: null + id: 2cccae35-0560-4b41-9ff4-07e59cef3bee + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-cs-target + reference: '' + 306d35e0-e9ea-45dc-abf8-fed4df11fdc9: !Template + answer_choices: null + id: 306d35e0-e9ea-45dc-abf8-fed4df11fdc9 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target + reference: '' + 3c095e27-6f4c-432d-b633-157a28e70a82: !Template + answer_choices: null + id: 3c095e27-6f4c-432d-b633-157a28e70a82 + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-target + reference: '' + 436eefe7-030f-4d9d-9791-c21c59456469: !Template + answer_choices: null + id: 436eefe7-030f-4d9d-9791-c21c59456469 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-src+target + reference: '' + 49994c64-6eb3-4874-8f90-da42f692a0a1: !Template + answer_choices: null + id: 49994c64-6eb3-4874-8f90-da42f692a0a1 + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target + reference: '' + 70ff8130-2515-4e14-a18c-d6eea4118893: !Template + answer_choices: null + id: 70ff8130-2515-4e14-a18c-d6eea4118893 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: '' + 8a7b8660-b64c-4207-a224-c2bb3c29e36f: !Template + answer_choices: null + id: 8a7b8660-b64c-4207-a224-c2bb3c29e36f + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-target + reference: '' + 9a400489-d8b6-4147-9595-538f63bbee8a: !Template + answer_choices: null + id: 9a400489-d8b6-4147-9595-538f63bbee8a + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + a3f15b8e-5398-42ba-aae9-13b7fd9e2285: !Template + answer_choices: null + id: a3f15b8e-5398-42ba-aae9-13b7fd9e2285 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + aac0fa41-9ea0-4e27-b2c0-384515f23fda: !Template + answer_choices: null + id: aac0fa41-9ea0-4e27-b2c0-384515f23fda + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' + ac0ab68b-84e9-4b9d-9e50-c842a51b2f77: !Template + answer_choices: null + id: ac0ab68b-84e9-4b9d-9e50-c842a51b2f77 + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target + reference: '' + d3da03a4-7845-4c6f-86ec-a551749d03d8: !Template + answer_choices: null + id: d3da03a4-7845-4c6f-86ec-a551749d03d8 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + d3f2c452-c838-4abf-9142-fbebfefed494: !Template + answer_choices: null + id: d3f2c452-c838-4abf-9142-fbebfefed494 + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target + reference: '' + d9386bcf-9dc9-448b-9f95-e01be14f1795: !Template + answer_choices: null + id: d9386bcf-9dc9-448b-9f95-e01be14f1795 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-source+target + reference: '' + ebe24ff7-03b0-4a37-9da3-c3c715949cd7: !Template + answer_choices: null + id: ebe24ff7-03b0-4a37-9da3-c3c715949cd7 + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target + reference: '' diff --git a/promptsource/templates/wmt16/de-en/templates.yaml b/promptsource/templates/wmt16/de-en/templates.yaml new file mode 100644 index 000000000..1bbf5f425 --- /dev/null +++ b/promptsource/templates/wmt16/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: de-en +templates: + 0cac8bad-ed93-405f-a810-cc4252b0148c: !Template + answer_choices: null + id: 0cac8bad-ed93-405f-a810-cc4252b0148c + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + 1348d039-2476-48ac-a2d2-7f66df53868d: !Template + answer_choices: null + id: 1348d039-2476-48ac-a2d2-7f66df53868d + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + 1606b0f9-35b7-40a9-a46c-f416de1f4c82: !Template + answer_choices: null + id: 1606b0f9-35b7-40a9-a46c-f416de1f4c82 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 1afad944-388b-4246-b83f-e10a3b3090b6: !Template + answer_choices: null + id: 1afad944-388b-4246-b83f-e10a3b3090b6 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + 229197cc-6240-4edb-b668-5efa298db15c: !Template + answer_choices: null + id: 229197cc-6240-4edb-b668-5efa298db15c + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 380d03fd-7b06-435f-a00d-e640f95ac357: !Template + answer_choices: null + id: 380d03fd-7b06-435f-a00d-e640f95ac357 + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' + 39d60c43-fd32-4078-9fdc-1fb6e09b7ce6: !Template + answer_choices: null + id: 39d60c43-fd32-4078-9fdc-1fb6e09b7ce6 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 63e8d2c7-578b-444d-a4dd-a1037ed9a748: !Template + answer_choices: null + id: 63e8d2c7-578b-444d-a4dd-a1037ed9a748 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 66367e82-d0a1-45e9-a04f-aa3937a5b5eb: !Template + answer_choices: null + id: 66367e82-d0a1-45e9-a04f-aa3937a5b5eb + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' + 66bd6d41-82fd-43ea-98c1-2e9e9ecdd3fd: !Template + answer_choices: null + id: 66bd6d41-82fd-43ea-98c1-2e9e9ecdd3fd + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + 702bbd13-29f7-4e83-95f5-09471856007b: !Template + answer_choices: null + id: 702bbd13-29f7-4e83-95f5-09471856007b + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + 738f96e7-1b92-42d0-b88c-99272e304e57: !Template + answer_choices: null + id: 738f96e7-1b92-42d0-b88c-99272e304e57 + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + 8bfca2fd-9bbd-42a0-b924-8a1fc891b152: !Template + answer_choices: null + id: 8bfca2fd-9bbd-42a0-b924-8a1fc891b152 + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + a52d0722-de2c-4cbc-89b1-430f9adf8b1d: !Template + answer_choices: null + id: a52d0722-de2c-4cbc-89b1-430f9adf8b1d + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + a7e7d7a3-791a-45de-91ac-62f4923296e9: !Template + answer_choices: null + id: a7e7d7a3-791a-45de-91ac-62f4923296e9 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + afff8a67-9085-401a-8de7-992a8cd630ad: !Template + answer_choices: null + id: afff8a67-9085-401a-8de7-992a8cd630ad + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + bb0bad5a-f598-48a4-871a-914da862f9fa: !Template + answer_choices: null + id: bb0bad5a-f598-48a4-871a-914da862f9fa + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + cc00a59b-9da3-4167-8a2d-c84670899941: !Template + answer_choices: null + id: cc00a59b-9da3-4167-8a2d-c84670899941 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + d0b83143-620c-4b18-9a27-c2dd9b82045b: !Template + answer_choices: null + id: d0b83143-620c-4b18-9a27-c2dd9b82045b + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + d9857ba6-8dea-40f0-ad61-757d21d08e05: !Template + answer_choices: null + id: d9857ba6-8dea-40f0-ad61-757d21d08e05 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + eb6e2d75-18ea-400b-baa5-650f7e3b6a37: !Template + answer_choices: null + id: eb6e2d75-18ea-400b-baa5-650f7e3b6a37 + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + f2f1d8c9-e5a0-4443-9343-c4e470346cba: !Template + answer_choices: null + id: f2f1d8c9-e5a0-4443-9343-c4e470346cba + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' diff --git a/promptsource/templates/wmt16/fi-en/templates.yaml b/promptsource/templates/wmt16/fi-en/templates.yaml new file mode 100644 index 000000000..4df5027f2 --- /dev/null +++ b/promptsource/templates/wmt16/fi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: fi-en +templates: + 009a2c37-87d1-4384-9301-a53b4ab811de: !Template + answer_choices: null + id: 009a2c37-87d1-4384-9301-a53b4ab811de + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-src+starget + reference: '' + 033f1d74-9bee-4851-8fd7-b087a7823c4e: !Template + answer_choices: null + id: 033f1d74-9bee-4851-8fd7-b087a7823c4e + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target + reference: '' + 13525c5d-3c9b-4554-a3bf-88acecb925d4: !Template + answer_choices: null + id: 13525c5d-3c9b-4554-a3bf-88acecb925d4 + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target + reference: '' + 1f8fd96b-3684-44a3-8bef-6e2ffc117347: !Template + answer_choices: null + id: 1f8fd96b-3684-44a3-8bef-6e2ffc117347 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target + reference: '' + 368b581c-c592-4bfd-a79e-2572d2c010be: !Template + answer_choices: null + id: 368b581c-c592-4bfd-a79e-2572d2c010be + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + 3bac8137-9ea7-4eb7-93af-44f61eedb495: !Template + answer_choices: null + id: 3bac8137-9ea7-4eb7-93af-44f61eedb495 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-src+target + reference: '' + 3dae3855-5014-4d1b-834c-713224698305: !Template + answer_choices: null + id: 3dae3855-5014-4d1b-834c-713224698305 + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-source+target + reference: '' + 5f36e461-251e-4690-9d23-3500380b965a: !Template + answer_choices: null + id: 5f36e461-251e-4690-9d23-3500380b965a + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: '' + 6f1bb4e0-5391-4357-87ca-d906594587ef: !Template + answer_choices: null + id: 6f1bb4e0-5391-4357-87ca-d906594587ef + jinja: 'What is the Finnish translation of: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-target + reference: '' + 83cadcf9-dc67-425b-9cb4-07c4bfc230d2: !Template + answer_choices: null + id: 83cadcf9-dc67-425b-9cb4-07c4bfc230d2 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-target + reference: '' + 8cb88018-e934-4717-aeb1-154314b7d640: !Template + answer_choices: null + id: 8cb88018-e934-4717-aeb1-154314b7d640 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target + reference: '' + 8f9765ec-86cf-49f2-8ba3-3329465977da: !Template + answer_choices: null + id: 8f9765ec-86cf-49f2-8ba3-3329465977da + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-target + reference: '' + 9b008796-65d0-4efc-83b8-69bed01e9ebc: !Template + answer_choices: null + id: 9b008796-65d0-4efc-83b8-69bed01e9ebc + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-target + reference: '' + 9bf2ce4f-5b6a-4778-8708-1e3f9b86cdff: !Template + answer_choices: null + id: 9bf2ce4f-5b6a-4778-8708-1e3f9b86cdff + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + a5dbdded-9234-4ebd-85d5-93ee37d382c8: !Template + answer_choices: null + id: a5dbdded-9234-4ebd-85d5-93ee37d382c8 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + ac4a34bc-d850-4ae1-8275-fc2af8eb0324: !Template + answer_choices: null + id: ac4a34bc-d850-4ae1-8275-fc2af8eb0324 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + ac6ad43d-d441-442b-b62c-ae11e1c197c8: !Template + answer_choices: null + id: ac6ad43d-d441-442b-b62c-ae11e1c197c8 + jinja: 'Translate Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + b92e677c-e5a0-475a-945a-41747d33ce7f: !Template + answer_choices: null + id: b92e677c-e5a0-475a-945a-41747d33ce7f + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-source+target + reference: '' + c06b082e-1028-4ec8-8ce1-6e6bd5ff9331: !Template + answer_choices: null + id: c06b082e-1028-4ec8-8ce1-6e6bd5ff9331 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-target + reference: '' + c714391d-7e6a-4621-8829-c4d08a70c64f: !Template + answer_choices: null + id: c714391d-7e6a-4621-8829-c4d08a70c64f + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-source+target + reference: '' + cdc5df34-d605-4eaf-9baa-cdc11952654f: !Template + answer_choices: null + id: cdc5df34-d605-4eaf-9baa-cdc11952654f + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-src+target + reference: '' + ff2ea6cd-1bb6-4192-8b29-8d9ab5fdb3fe: !Template + answer_choices: null + id: ff2ea6cd-1bb6-4192-8b29-8d9ab5fdb3fe + jinja: 'How do you say {{translation["fi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fi-en-target + reference: '' diff --git a/promptsource/templates/wmt16/ro-en/templates.yaml b/promptsource/templates/wmt16/ro-en/templates.yaml new file mode 100644 index 000000000..be38d7df7 --- /dev/null +++ b/promptsource/templates/wmt16/ro-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: ro-en +templates: + 2d942085-1c5a-4b3d-956a-c50385ccb2dc: !Template + answer_choices: null + id: 2d942085-1c5a-4b3d-956a-c50385ccb2dc + jinja: 'How do you say {{translation["en"]}} in Romanian? + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ro-target + reference: '' + 2de0143d-57c2-4379-8b87-f11cfb386039: !Template + answer_choices: null + id: 2de0143d-57c2-4379-8b87-f11cfb386039 + jinja: 'What is the Romanian translation of: {{translation["en"]}} + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ro-target + reference: '' + 2edfd6ed-0233-4062-aa9c-66bef54b2a34: !Template + answer_choices: null + id: 2edfd6ed-0233-4062-aa9c-66bef54b2a34 + jinja: 'If the original version says: {{translation["ro"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ro-en-target + reference: '' + 48998b22-4bad-4a98-8d0d-ad90584845f2: !Template + answer_choices: null + id: 48998b22-4bad-4a98-8d0d-ad90584845f2 + jinja: 'Translate Romanian into English: {{translation["ro"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ro-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 4cd03022-c0e5-4d80-810d-f99d345d0ce4: !Template + answer_choices: null + id: 4cd03022-c0e5-4d80-810d-f99d345d0ce4 + jinja: 'Translate English into Romanian: {{translation["en"]}} + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ro-src+starget + reference: '' + 4d0f8d04-3149-4696-9764-d194afa31936: !Template + answer_choices: null + id: 4d0f8d04-3149-4696-9764-d194afa31936 + jinja: 'Translate into English: {{translation["ro"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ro-en-target + reference: '' + 56382179-2a48-446d-8de3-4701a888d122: !Template + answer_choices: null + id: 56382179-2a48-446d-8de3-4701a888d122 + jinja: '{{translation["ro"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ro-en-target + reference: '' + 6179fada-a56f-4a46-aec7-4b637a77f24f: !Template + answer_choices: null + id: 6179fada-a56f-4a46-aec7-4b637a77f24f + jinja: 'What is the English translation of : {{translation["ro"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ro-en-target + reference: '' + 68bb72ef-c866-467f-abe9-d6c89f58175a: !Template + answer_choices: null + id: 68bb72ef-c866-467f-abe9-d6c89f58175a + jinja: 'If the original version says: {{translation["en"]}}; then the Romanian + version should say: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ro-target + reference: '' + 6ed88276-594a-4c97-a8c0-b637e154d5b1: !Template + answer_choices: null + id: 6ed88276-594a-4c97-a8c0-b637e154d5b1 + jinja: 'Given the following passage: {{translation["ro"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ro-en-target + reference: '' + 7bffa288-bdd0-4b0f-8738-44540b1154e7: !Template + answer_choices: null + id: 7bffa288-bdd0-4b0f-8738-44540b1154e7 + jinja: 'What is the Romanian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ro-src+target + reference: '' + 8142e62d-c4a6-4cf9-9d66-59ba23aa177e: !Template + answer_choices: null + id: 8142e62d-c4a6-4cf9-9d66-59ba23aa177e + jinja: 'How do you say {{translation["ro"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ro-en-target + reference: '' + 8871ac3e-f8a1-491f-89cd-366d6133cb76: !Template + answer_choices: null + id: 8871ac3e-f8a1-491f-89cd-366d6133cb76 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Romanian translation is: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ro-source+target + reference: '' + 899f7360-9314-4057-a6ba-35b1466d22c1: !Template + answer_choices: null + id: 899f7360-9314-4057-a6ba-35b1466d22c1 + jinja: ' {{translation["en"]}} translates into Romanian as: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ro-target + reference: '' + a0726d07-8f0a-4b1e-882f-b10368bfa0f0: !Template + answer_choices: null + id: a0726d07-8f0a-4b1e-882f-b10368bfa0f0 + jinja: 'Given the following passage: {{translation["en"]}} , a good Romanian translation + is: ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ro-target + reference: '' + b5c54c2c-1232-4779-bb58-919f81d8cfd6: !Template + answer_choices: null + id: b5c54c2c-1232-4779-bb58-919f81d8cfd6 + jinja: 'Translate into Romanian: {{translation["en"]}} + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ro-target + reference: '' + b92197f6-88bf-4375-8d43-48a784d4fd98: !Template + answer_choices: null + id: b92197f6-88bf-4375-8d43-48a784d4fd98 + jinja: 'Given the following source text in Romanian: {{translation["ro"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ro-en-source+target + reference: '' + da7917f0-aad3-4a6b-83b1-6c94c7a5c3e9: !Template + answer_choices: null + id: da7917f0-aad3-4a6b-83b1-6c94c7a5c3e9 + jinja: 'English: {{translation["en"]}} translates into Romanian as: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ro-source+target + reference: '' + dc349c72-3b44-4fa6-8998-099c86e07ecd: !Template + answer_choices: null + id: dc349c72-3b44-4fa6-8998-099c86e07ecd + jinja: 'Romanian: {{translation["ro"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ro-en-source+target + reference: '' + dc531152-e769-4b15-8c08-07829e7c7da7: !Template + answer_choices: null + id: dc531152-e769-4b15-8c08-07829e7c7da7 + jinja: 'If the English version says: {{translation["en"]}}; then the Romanian + version should say: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ro-source+target + reference: '' + e61371c7-6627-4578-bcae-9127a49ea772: !Template + answer_choices: null + id: e61371c7-6627-4578-bcae-9127a49ea772 + jinja: 'What is the English translation of the Romanian sentence: {{translation["ro"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ro-en-src+target + reference: '' + f38ceb60-50e2-4b78-9c80-2bea7a5231c0: !Template + answer_choices: null + id: f38ceb60-50e2-4b78-9c80-2bea7a5231c0 + jinja: 'If the Romanian version says: {{translation["ro"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ro-en-source+target + reference: '' diff --git a/promptsource/templates/wmt16/ru-en/templates.yaml b/promptsource/templates/wmt16/ru-en/templates.yaml new file mode 100644 index 000000000..948310c91 --- /dev/null +++ b/promptsource/templates/wmt16/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: ru-en +templates: + 12af9256-262a-4b74-91be-ff32c1afbd3c: !Template + answer_choices: null + id: 12af9256-262a-4b74-91be-ff32c1afbd3c + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + 1c5d3480-e1a4-44f1-9549-d5b3d5d8490e: !Template + answer_choices: null + id: 1c5d3480-e1a4-44f1-9549-d5b3d5d8490e + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' + 2069bf34-0c3f-4512-b87a-71a4e051e10e: !Template + answer_choices: null + id: 2069bf34-0c3f-4512-b87a-71a4e051e10e + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' + 267de81e-0791-419f-b9be-f9461b70fc4b: !Template + answer_choices: null + id: 267de81e-0791-419f-b9be-f9461b70fc4b + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + 2825d558-0af8-4259-9258-372254a5f7c6: !Template + answer_choices: null + id: 2825d558-0af8-4259-9258-372254a5f7c6 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + 2c197bee-b468-4fa8-a2a9-07d023a86e36: !Template + answer_choices: null + id: 2c197bee-b468-4fa8-a2a9-07d023a86e36 + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + 3e92539c-1589-4684-a0d8-fef5451727d0: !Template + answer_choices: null + id: 3e92539c-1589-4684-a0d8-fef5451727d0 + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' + 43e99736-f6f3-4b11-aec1-3629f7b4d924: !Template + answer_choices: null + id: 43e99736-f6f3-4b11-aec1-3629f7b4d924 + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + 4e5de29d-81f1-4742-8c47-4a7185639afd: !Template + answer_choices: null + id: 4e5de29d-81f1-4742-8c47-4a7185639afd + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + 4f3a5dd7-6180-4140-b70d-f455dc5b9908: !Template + answer_choices: null + id: 4f3a5dd7-6180-4140-b70d-f455dc5b9908 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + 4f9cc10f-b20f-4fbb-8677-717a6dd075c2: !Template + answer_choices: null + id: 4f9cc10f-b20f-4fbb-8677-717a6dd075c2 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + 60858979-aa78-427a-9a7f-d60d92ae4dc2: !Template + answer_choices: null + id: 60858979-aa78-427a-9a7f-d60d92ae4dc2 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + 635bee5f-d63b-465e-adf7-b21528af3ebe: !Template + answer_choices: null + id: 635bee5f-d63b-465e-adf7-b21528af3ebe + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 6828bba2-fb1f-4ffb-b775-2d8cde0e24a9: !Template + answer_choices: null + id: 6828bba2-fb1f-4ffb-b775-2d8cde0e24a9 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + 74ab5d76-1cdc-4864-b10f-6d40f9676717: !Template + answer_choices: null + id: 74ab5d76-1cdc-4864-b10f-6d40f9676717 + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + 75dd6cf8-1b57-4976-9691-2e9e0ddc5c37: !Template + answer_choices: null + id: 75dd6cf8-1b57-4976-9691-2e9e0ddc5c37 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + 8c18f538-d59a-4bcc-bab7-b655ba847372: !Template + answer_choices: null + id: 8c18f538-d59a-4bcc-bab7-b655ba847372 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + cd6e3dbd-36fa-4774-9bac-78d8d2b50a0d: !Template + answer_choices: null + id: cd6e3dbd-36fa-4774-9bac-78d8d2b50a0d + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + eb4dcd70-1bf3-4a96-abaf-38521fa957ee: !Template + answer_choices: null + id: eb4dcd70-1bf3-4a96-abaf-38521fa957ee + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + edef1b19-1f34-4a6c-bc06-516a0f5baab4: !Template + answer_choices: null + id: edef1b19-1f34-4a6c-bc06-516a0f5baab4 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' + ee42c631-6217-4bf5-b0a4-0a1dfa2c4651: !Template + answer_choices: null + id: ee42c631-6217-4bf5-b0a4-0a1dfa2c4651 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + f31e5d99-55c7-4369-a07c-0efa4d5e4ef3: !Template + answer_choices: null + id: f31e5d99-55c7-4369-a07c-0efa4d5e4ef3 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' diff --git a/promptsource/templates/wmt16/tr-en/templates.yaml b/promptsource/templates/wmt16/tr-en/templates.yaml new file mode 100644 index 000000000..ca63b34b4 --- /dev/null +++ b/promptsource/templates/wmt16/tr-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt16 +subset: tr-en +templates: + 0930ab6b-c545-45cb-afd2-7776011bf61b: !Template + answer_choices: null + id: 0930ab6b-c545-45cb-afd2-7776011bf61b + jinja: 'English: {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-source+target + reference: '' + 0f4540f4-8b32-4ff0-b731-e8ac6306525b: !Template + answer_choices: null + id: 0f4540f4-8b32-4ff0-b731-e8ac6306525b + jinja: 'How do you say {{translation["tr"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-tr-en-target + reference: '' + 151edb16-4e30-4495-ab94-0506a3795f44: !Template + answer_choices: null + id: 151edb16-4e30-4495-ab94-0506a3795f44 + jinja: 'Translate Turkish into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 2cf4e3c4-a1f5-48c1-b87a-5625934d351f: !Template + answer_choices: null + id: 2cf4e3c4-a1f5-48c1-b87a-5625934d351f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-source+target + reference: '' + 527c953c-a8b6-4876-b7d1-d7fe67edabbb: !Template + answer_choices: null + id: 527c953c-a8b6-4876-b7d1-d7fe67edabbb + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-src+target + reference: '' + 5f09b297-0c8a-4d66-84cb-36e9cd19f9f9: !Template + answer_choices: null + id: 5f09b297-0c8a-4d66-84cb-36e9cd19f9f9 + jinja: '{{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-target + reference: '' + 61637dbe-1c10-47f6-a466-ddb8f1964c06: !Template + answer_choices: null + id: 61637dbe-1c10-47f6-a466-ddb8f1964c06 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-source+target + reference: '' + 63d5e992-f53e-4e9f-8205-923084e7cbeb: !Template + answer_choices: null + id: 63d5e992-f53e-4e9f-8205-923084e7cbeb + jinja: 'What is the Turkish translation of: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-target + reference: '' + 779758cf-ed3e-412d-8862-2accc1bd8b4a: !Template + answer_choices: null + id: 779758cf-ed3e-412d-8862-2accc1bd8b4a + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-target + reference: '' + 7e877011-b2a7-4413-8d9e-1c4ed886ad90: !Template + answer_choices: null + id: 7e877011-b2a7-4413-8d9e-1c4ed886ad90 + jinja: 'Translate into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-target + reference: '' + 7fbb5eea-a23f-4613-bb0d-e07d90d7b688: !Template + answer_choices: null + id: 7fbb5eea-a23f-4613-bb0d-e07d90d7b688 + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-source+target + reference: '' + 8c33de3d-b2b8-49c7-b453-a37ccdb17fce: !Template + answer_choices: null + id: 8c33de3d-b2b8-49c7-b453-a37ccdb17fce + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-source+target + reference: '' + 8fc26430-12b6-425b-883d-cb9cc2d0535d: !Template + answer_choices: null + id: 8fc26430-12b6-425b-883d-cb9cc2d0535d + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-target + reference: '' + a0912822-af17-4949-992e-5979481c82d0: !Template + answer_choices: null + id: a0912822-af17-4949-992e-5979481c82d0 + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-src+target + reference: '' + ad1a4010-fb06-4257-8789-4ea6550963a8: !Template + answer_choices: null + id: ad1a4010-fb06-4257-8789-4ea6550963a8 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-target + reference: '' + b1f8bdf0-1410-4de1-983b-aee71e1f202b: !Template + answer_choices: null + id: b1f8bdf0-1410-4de1-983b-aee71e1f202b + jinja: ' {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-target + reference: '' + be71bc64-4d1e-432b-ae21-344bf39b577a: !Template + answer_choices: null + id: be71bc64-4d1e-432b-ae21-344bf39b577a + jinja: 'Translate English into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-src+starget + reference: '' + d724d6f8-bc52-439f-898a-a32ea7317377: !Template + answer_choices: null + id: d724d6f8-bc52-439f-898a-a32ea7317377 + jinja: 'What is the English translation of : {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-target + reference: '' + d8bbecc3-f551-4931-aa64-d01d2b5d0630: !Template + answer_choices: null + id: d8bbecc3-f551-4931-aa64-d01d2b5d0630 + jinja: 'Translate into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-target + reference: '' + daad9f22-4bcf-41cb-9936-b17ff52a47d6: !Template + answer_choices: null + id: daad9f22-4bcf-41cb-9936-b17ff52a47d6 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-source+target + reference: '' + e8548697-38a5-4076-a11e-ac3be8f9ea48: !Template + answer_choices: null + id: e8548697-38a5-4076-a11e-ac3be8f9ea48 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-target + reference: '' + f51b4cf6-bd87-4bbc-92f9-0f77b2834e48: !Template + answer_choices: null + id: f51b4cf6-bd87-4bbc-92f9-0f77b2834e48 + jinja: 'How do you say {{translation["en"]}} in Turkish? + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-tr-target + reference: '' diff --git a/promptsource/templates/wmt17/cs-en/templates.yaml b/promptsource/templates/wmt17/cs-en/templates.yaml new file mode 100644 index 000000000..471e828f1 --- /dev/null +++ b/promptsource/templates/wmt17/cs-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: cs-en +templates: + 01a9b4cc-3e7d-4472-bc07-423d85faad52: !Template + answer_choices: null + id: 01a9b4cc-3e7d-4472-bc07-423d85faad52 + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-target + reference: '' + 0b196075-faee-4e54-be5c-28cde617bcec: !Template + answer_choices: null + id: 0b196075-faee-4e54-be5c-28cde617bcec + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-source+target + reference: '' + 1deb5592-6867-421c-9421-e2a74a6ebf11: !Template + answer_choices: null + id: 1deb5592-6867-421c-9421-e2a74a6ebf11 + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target + reference: '' + 2aaea840-09fa-4496-9ac2-3ce92b739ccb: !Template + answer_choices: null + id: 2aaea840-09fa-4496-9ac2-3ce92b739ccb + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target + reference: '' + 40e25397-0a8e-4289-b28a-aeaa1f23ddb2: !Template + answer_choices: null + id: 40e25397-0a8e-4289-b28a-aeaa1f23ddb2 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: '' + 59d1355f-c705-4267-95be-9e98bb5fbb90: !Template + answer_choices: null + id: 59d1355f-c705-4267-95be-9e98bb5fbb90 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + 61056233-ad4c-4226-9854-9e4a46cfea3f: !Template + answer_choices: null + id: 61056233-ad4c-4226-9854-9e4a46cfea3f + jinja: 'English: {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-source+target + reference: '' + 6bec36bf-a843-488c-8843-cae39b644efa: !Template + answer_choices: null + id: 6bec36bf-a843-488c-8843-cae39b644efa + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-target + reference: '' + 6e96ba15-9c50-4770-a8df-d4d2e04885cd: !Template + answer_choices: null + id: 6e96ba15-9c50-4770-a8df-d4d2e04885cd + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + 89c49a56-cb7a-4d19-bed9-a4ef79476cca: !Template + answer_choices: null + id: 89c49a56-cb7a-4d19-bed9-a4ef79476cca + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' + 8ffa0081-e870-47e9-99cf-ef649372507b: !Template + answer_choices: null + id: 8ffa0081-e870-47e9-99cf-ef649372507b + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-cs-target + reference: '' + 930e9528-823c-4d19-8a53-4d1a1534a06d: !Template + answer_choices: null + id: 930e9528-823c-4d19-8a53-4d1a1534a06d + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-target + reference: '' + 9f3279de-22db-45e9-989b-22acebdf7f02: !Template + answer_choices: null + id: 9f3279de-22db-45e9-989b-22acebdf7f02 + jinja: 'Translate English into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-src+starget + reference: '' + a5d42259-2578-4f0f-ab0d-be7c76122ed6: !Template + answer_choices: null + id: a5d42259-2578-4f0f-ab0d-be7c76122ed6 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-src+target + reference: '' + ab56af3e-f85b-42e5-a4de-dd791093530d: !Template + answer_choices: null + id: ab56af3e-f85b-42e5-a4de-dd791093530d + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target + reference: '' + aca1630b-8e84-481f-a3a1-4bdaaaa6fcda: !Template + answer_choices: null + id: aca1630b-8e84-481f-a3a1-4bdaaaa6fcda + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target + reference: '' + b847c014-264b-414a-9e84-1a087daefcb9: !Template + answer_choices: null + id: b847c014-264b-414a-9e84-1a087daefcb9 + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target + reference: '' + ba823677-d7cd-4e42-9e55-120ee6b07e66: !Template + answer_choices: null + id: ba823677-d7cd-4e42-9e55-120ee6b07e66 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-src+target + reference: '' + bd8b3c35-291e-4b37-b270-c40234c11324: !Template + answer_choices: null + id: bd8b3c35-291e-4b37-b270-c40234c11324 + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e2a453da-ab63-4ce4-85fe-6173f3eff740: !Template + answer_choices: null + id: e2a453da-ab63-4ce4-85fe-6173f3eff740 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + f98dbcb7-9da6-4903-893a-c2dd3337f2fe: !Template + answer_choices: null + id: f98dbcb7-9da6-4903-893a-c2dd3337f2fe + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target + reference: '' + fed191b4-e6c1-4d6f-86dc-fcedbccadcf2: !Template + answer_choices: null + id: fed191b4-e6c1-4d6f-86dc-fcedbccadcf2 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-source+target + reference: '' diff --git a/promptsource/templates/wmt17/de-en/templates.yaml b/promptsource/templates/wmt17/de-en/templates.yaml new file mode 100644 index 000000000..3d966b2d9 --- /dev/null +++ b/promptsource/templates/wmt17/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: de-en +templates: + 0412ba2f-a534-40bd-818f-9417b357eeea: !Template + answer_choices: null + id: 0412ba2f-a534-40bd-818f-9417b357eeea + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1497ff4a-f007-4761-a8f1-4b01ac2fdb3a: !Template + answer_choices: null + id: 1497ff4a-f007-4761-a8f1-4b01ac2fdb3a + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + 28cee50e-f52b-4bc4-a295-c7ad574012b1: !Template + answer_choices: null + id: 28cee50e-f52b-4bc4-a295-c7ad574012b1 + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + 3788ad81-f204-4658-a5f5-95dd2273a0e5: !Template + answer_choices: null + id: 3788ad81-f204-4658-a5f5-95dd2273a0e5 + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + 4b443fc4-3f35-4420-ba69-9b3cc39c2377: !Template + answer_choices: null + id: 4b443fc4-3f35-4420-ba69-9b3cc39c2377 + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + 60e576b9-62f0-40b1-a55c-bbcdc9600ce8: !Template + answer_choices: null + id: 60e576b9-62f0-40b1-a55c-bbcdc9600ce8 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + 6e58dba7-4a9c-49f9-87cc-833941ffb953: !Template + answer_choices: null + id: 6e58dba7-4a9c-49f9-87cc-833941ffb953 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 807a15d9-c3f7-42c6-9081-242a67e5f36f: !Template + answer_choices: null + id: 807a15d9-c3f7-42c6-9081-242a67e5f36f + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 883446e1-6d30-4279-87bd-f977e501d70f: !Template + answer_choices: null + id: 883446e1-6d30-4279-87bd-f977e501d70f + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' + 8a2e744b-333a-4054-8d92-a344d236b57e: !Template + answer_choices: null + id: 8a2e744b-333a-4054-8d92-a344d236b57e + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + 8a35b90d-e148-44d5-b0d4-437af5c00ace: !Template + answer_choices: null + id: 8a35b90d-e148-44d5-b0d4-437af5c00ace + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 8c78b498-8cec-44cc-a9fd-2ea72609f201: !Template + answer_choices: null + id: 8c78b498-8cec-44cc-a9fd-2ea72609f201 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + 95667fc5-aaff-4675-828c-79be2badfee7: !Template + answer_choices: null + id: 95667fc5-aaff-4675-828c-79be2badfee7 + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + a1a4e4f8-fec2-4802-b467-1a3110d9c732: !Template + answer_choices: null + id: a1a4e4f8-fec2-4802-b467-1a3110d9c732 + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' + b78030c3-cef3-4aa8-ac05-ec753e24fdc3: !Template + answer_choices: null + id: b78030c3-cef3-4aa8-ac05-ec753e24fdc3 + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + cb1bee80-252f-4bcb-8657-b1b063dd014b: !Template + answer_choices: null + id: cb1bee80-252f-4bcb-8657-b1b063dd014b + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + d350fec6-66c5-45f5-b9de-701faa4e37b8: !Template + answer_choices: null + id: d350fec6-66c5-45f5-b9de-701faa4e37b8 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + e7a1e365-ac55-48b7-9c3f-a13551a29e0e: !Template + answer_choices: null + id: e7a1e365-ac55-48b7-9c3f-a13551a29e0e + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + ed7eba49-70b4-41ce-bb60-8ef299950dcf: !Template + answer_choices: null + id: ed7eba49-70b4-41ce-bb60-8ef299950dcf + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + f0c3c5ae-cca6-434c-b54e-12c1b1440a69: !Template + answer_choices: null + id: f0c3c5ae-cca6-434c-b54e-12c1b1440a69 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + f1febb5f-bb1f-4aad-aad8-fe1b89ee329b: !Template + answer_choices: null + id: f1febb5f-bb1f-4aad-aad8-fe1b89ee329b + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + f6ff0677-5068-4e1f-940a-86ef559a464c: !Template + answer_choices: null + id: f6ff0677-5068-4e1f-940a-86ef559a464c + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' diff --git a/promptsource/templates/wmt17/fi-en/templates.yaml b/promptsource/templates/wmt17/fi-en/templates.yaml new file mode 100644 index 000000000..eac87c0ab --- /dev/null +++ b/promptsource/templates/wmt17/fi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: fi-en +templates: + 0088b4f4-fe6e-44b4-9740-62e258676a9a: !Template + answer_choices: null + id: 0088b4f4-fe6e-44b4-9740-62e258676a9a + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + 15909dca-c95f-4395-be59-f1c67498422e: !Template + answer_choices: null + id: 15909dca-c95f-4395-be59-f1c67498422e + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-src+starget + reference: '' + 27078522-e6b8-419f-aab5-3f37484790ef: !Template + answer_choices: null + id: 27078522-e6b8-419f-aab5-3f37484790ef + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + 2af8ac5d-cf62-4b4a-9ded-1dbc276bc406: !Template + answer_choices: null + id: 2af8ac5d-cf62-4b4a-9ded-1dbc276bc406 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-source+target + reference: '' + 2f61c246-5222-49d2-9e03-bd66cf249e76: !Template + answer_choices: null + id: 2f61c246-5222-49d2-9e03-bd66cf249e76 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + 2fa36657-1938-4959-81e5-eca24f504804: !Template + answer_choices: null + id: 2fa36657-1938-4959-81e5-eca24f504804 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-target + reference: '' + 46a26437-fab2-4cc5-b7e2-968f29b1b606: !Template + answer_choices: null + id: 46a26437-fab2-4cc5-b7e2-968f29b1b606 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-target + reference: '' + 48df8b43-caa8-4a08-84ed-b8508a134744: !Template + answer_choices: null + id: 48df8b43-caa8-4a08-84ed-b8508a134744 + jinja: 'What is the Finnish translation of: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-target + reference: '' + 78571171-181d-43b5-aed4-c64dbbe0a88f: !Template + answer_choices: null + id: 78571171-181d-43b5-aed4-c64dbbe0a88f + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-target + reference: '' + 7c4fa3c5-5edd-4c08-ad9d-27a6149493b1: !Template + answer_choices: null + id: 7c4fa3c5-5edd-4c08-ad9d-27a6149493b1 + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-source+target + reference: '' + 83347cec-5e58-4242-b747-8ada796193a2: !Template + answer_choices: null + id: 83347cec-5e58-4242-b747-8ada796193a2 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + 8783347e-1618-4692-8cc3-09e382b9048c: !Template + answer_choices: null + id: 8783347e-1618-4692-8cc3-09e382b9048c + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-src+target + reference: '' + 90ebc53b-7099-48cd-8682-ea591c7b7270: !Template + answer_choices: null + id: 90ebc53b-7099-48cd-8682-ea591c7b7270 + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target + reference: '' + 96b60ec8-4385-466c-a9b3-b3f0b6bcedc5: !Template + answer_choices: null + id: 96b60ec8-4385-466c-a9b3-b3f0b6bcedc5 + jinja: 'How do you say {{translation["fi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fi-en-target + reference: '' + 9876be1e-30e4-42ae-a9cc-d6e657dd2fa4: !Template + answer_choices: null + id: 9876be1e-30e4-42ae-a9cc-d6e657dd2fa4 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target + reference: '' + a3d06482-31e6-4f96-9531-f312236e3ead: !Template + answer_choices: null + id: a3d06482-31e6-4f96-9531-f312236e3ead + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-src+target + reference: '' + b0442673-6b81-44cf-ab0a-d2a922873a16: !Template + answer_choices: null + id: b0442673-6b81-44cf-ab0a-d2a922873a16 + jinja: 'Translate Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + c921267d-4041-442e-823d-340386da1d4e: !Template + answer_choices: null + id: c921267d-4041-442e-823d-340386da1d4e + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target + reference: '' + f1790f28-f93b-43d6-ba9d-8118bb6e5330: !Template + answer_choices: null + id: f1790f28-f93b-43d6-ba9d-8118bb6e5330 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target + reference: '' + f2590e75-f5ff-48fc-a01c-4694163ecdfd: !Template + answer_choices: null + id: f2590e75-f5ff-48fc-a01c-4694163ecdfd + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: '' + f7ca1c96-3a6d-4b76-843c-902658308396: !Template + answer_choices: null + id: f7ca1c96-3a6d-4b76-843c-902658308396 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-target + reference: '' + ff0bff92-8f1b-422c-9f4a-84e5d5808301: !Template + answer_choices: null + id: ff0bff92-8f1b-422c-9f4a-84e5d5808301 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-source+target + reference: '' diff --git a/promptsource/templates/wmt17/lv-en/templates.yaml b/promptsource/templates/wmt17/lv-en/templates.yaml new file mode 100644 index 000000000..f5138850a --- /dev/null +++ b/promptsource/templates/wmt17/lv-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: lv-en +templates: + 009646b7-7723-4d3b-b766-b4fe191b3bdd: !Template + answer_choices: null + id: 009646b7-7723-4d3b-b766-b4fe191b3bdd + jinja: 'If the original version says: {{translation["lv"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-lv-en-target + reference: '' + 07f78f8d-e5f6-41ea-a48d-237f4c917990: !Template + answer_choices: null + id: 07f78f8d-e5f6-41ea-a48d-237f4c917990 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Latvian translation is: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lv-source+target + reference: '' + 1997ff46-074f-4154-bcd7-5131ea1ed7d0: !Template + answer_choices: null + id: 1997ff46-074f-4154-bcd7-5131ea1ed7d0 + jinja: 'If the original version says: {{translation["en"]}}; then the Latvian + version should say: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-lv-target + reference: '' + 4b504847-a119-4d66-b3c6-4584c9f2e74b: !Template + answer_choices: null + id: 4b504847-a119-4d66-b3c6-4584c9f2e74b + jinja: 'What is the English translation of : {{translation["lv"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-lv-en-target + reference: '' + 630b8b7f-2335-4b2d-9c21-936a7a5597c4: !Template + answer_choices: null + id: 630b8b7f-2335-4b2d-9c21-936a7a5597c4 + jinja: 'What is the Latvian translation of: {{translation["en"]}} + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-lv-target + reference: '' + 6e911898-76f2-4686-bd7f-8db625bc3d57: !Template + answer_choices: null + id: 6e911898-76f2-4686-bd7f-8db625bc3d57 + jinja: 'Given the following passage: {{translation["lv"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-lv-en-target + reference: '' + 788254a0-c30d-4cf7-997d-b4af2b6c3f74: !Template + answer_choices: null + id: 788254a0-c30d-4cf7-997d-b4af2b6c3f74 + jinja: 'Given the following passage: {{translation["en"]}} , a good Latvian translation + is: ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lv-target + reference: '' + 7b9214a6-7bb2-4965-82dc-26c878b6b14f: !Template + answer_choices: null + id: 7b9214a6-7bb2-4965-82dc-26c878b6b14f + jinja: 'Translate into Latvian: {{translation["en"]}} + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-lv-target + reference: '' + 7eca2651-caca-4d2e-825d-b5c580d09a62: !Template + answer_choices: null + id: 7eca2651-caca-4d2e-825d-b5c580d09a62 + jinja: 'If the Latvian version says: {{translation["lv"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-lv-en-source+target + reference: '' + 973aab0a-a230-497f-8f4f-e41adba60a97: !Template + answer_choices: null + id: 973aab0a-a230-497f-8f4f-e41adba60a97 + jinja: ' {{translation["en"]}} translates into Latvian as: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-lv-target + reference: '' + a549c27c-acee-471e-bb6e-926f4eb780d8: !Template + answer_choices: null + id: a549c27c-acee-471e-bb6e-926f4eb780d8 + jinja: '{{translation["lv"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lv-en-target + reference: '' + ae4d88ef-7fa4-49f2-ab77-36a95b28d3d1: !Template + answer_choices: null + id: ae4d88ef-7fa4-49f2-ab77-36a95b28d3d1 + jinja: 'English: {{translation["en"]}} translates into Latvian as: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-lv-source+target + reference: '' + aff14994-4e76-44fa-ad8e-f2ca5959ff39: !Template + answer_choices: null + id: aff14994-4e76-44fa-ad8e-f2ca5959ff39 + jinja: 'How do you say {{translation["lv"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-lv-en-target + reference: '' + b23244c9-21ee-4c6f-83c7-1c2a2c75e1a0: !Template + answer_choices: null + id: b23244c9-21ee-4c6f-83c7-1c2a2c75e1a0 + jinja: 'Translate Latvian into English: {{translation["lv"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-lv-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + c24231a3-1294-4ea8-8bb6-789b6fc60868: !Template + answer_choices: null + id: c24231a3-1294-4ea8-8bb6-789b6fc60868 + jinja: 'Given the following source text in Latvian: {{translation["lv"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-lv-en-source+target + reference: '' + ceb16807-043a-4298-8766-0f60b710b364: !Template + answer_choices: null + id: ceb16807-043a-4298-8766-0f60b710b364 + jinja: 'If the English version says: {{translation["en"]}}; then the Latvian version + should say: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-lv-source+target + reference: '' + d065a3f5-21ef-4f64-b2a1-93be707dadcd: !Template + answer_choices: null + id: d065a3f5-21ef-4f64-b2a1-93be707dadcd + jinja: 'What is the Latvian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-lv-src+target + reference: '' + d2acabb3-9a40-4481-8cc5-9be87ef0be1f: !Template + answer_choices: null + id: d2acabb3-9a40-4481-8cc5-9be87ef0be1f + jinja: 'Translate English into Latvian: {{translation["en"]}} + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-lv-src+starget + reference: '' + db6c79b7-4d5b-43e5-bd40-9a0a557cd67f: !Template + answer_choices: null + id: db6c79b7-4d5b-43e5-bd40-9a0a557cd67f + jinja: 'What is the English translation of the Latvian sentence: {{translation["lv"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-lv-en-src+target + reference: '' + dbdb9686-a851-46bf-8364-bf85eea6d5f0: !Template + answer_choices: null + id: dbdb9686-a851-46bf-8364-bf85eea6d5f0 + jinja: 'Latvian: {{translation["lv"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lv-en-source+target + reference: '' + dde440d5-0458-4221-86b5-0315ef4505bb: !Template + answer_choices: null + id: dde440d5-0458-4221-86b5-0315ef4505bb + jinja: 'Translate into English: {{translation["lv"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-lv-en-target + reference: '' + fface4f5-aa4d-4ede-b395-08e20defe516: !Template + answer_choices: null + id: fface4f5-aa4d-4ede-b395-08e20defe516 + jinja: 'How do you say {{translation["en"]}} in Latvian? + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-lv-target + reference: '' diff --git a/promptsource/templates/wmt17/ru-en/templates.yaml b/promptsource/templates/wmt17/ru-en/templates.yaml new file mode 100644 index 000000000..7e223828e --- /dev/null +++ b/promptsource/templates/wmt17/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: ru-en +templates: + 01f14ae4-4289-447a-b769-533bc6b2cfa4: !Template + answer_choices: null + id: 01f14ae4-4289-447a-b769-533bc6b2cfa4 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + 0b70acb0-fc6c-44e7-93e4-2472c8fd9610: !Template + answer_choices: null + id: 0b70acb0-fc6c-44e7-93e4-2472c8fd9610 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + 0c082698-0676-4ce9-bd42-4985fc6ab3bc: !Template + answer_choices: null + id: 0c082698-0676-4ce9-bd42-4985fc6ab3bc + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + 0e9f93b2-0de5-4232-8486-01fc765ebc45: !Template + answer_choices: null + id: 0e9f93b2-0de5-4232-8486-01fc765ebc45 + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' + 0ef81131-531c-43f9-89e8-1f237fdbee5b: !Template + answer_choices: null + id: 0ef81131-531c-43f9-89e8-1f237fdbee5b + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + 1897c494-b380-45ef-b321-eee93ccc8e77: !Template + answer_choices: null + id: 1897c494-b380-45ef-b321-eee93ccc8e77 + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 20ce6360-fdc0-40e6-9fa7-fe710a24f90a: !Template + answer_choices: null + id: 20ce6360-fdc0-40e6-9fa7-fe710a24f90a + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' + 47870e82-5ed7-4a76-a94d-f9530cf31dc1: !Template + answer_choices: null + id: 47870e82-5ed7-4a76-a94d-f9530cf31dc1 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + 68e43b6a-ae41-4d30-b088-b438cf1305a2: !Template + answer_choices: null + id: 68e43b6a-ae41-4d30-b088-b438cf1305a2 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + 759f64ca-5f6c-4266-8af1-77e4cecb0003: !Template + answer_choices: null + id: 759f64ca-5f6c-4266-8af1-77e4cecb0003 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + 76d43586-31d4-4203-be38-e16d4a8d7ea6: !Template + answer_choices: null + id: 76d43586-31d4-4203-be38-e16d4a8d7ea6 + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + 79ad759f-3888-4e17-a5aa-ab9060c1ba8b: !Template + answer_choices: null + id: 79ad759f-3888-4e17-a5aa-ab9060c1ba8b + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + 871fe390-9b79-4c26-b006-d1e75a2b4ba3: !Template + answer_choices: null + id: 871fe390-9b79-4c26-b006-d1e75a2b4ba3 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + 891fa5eb-33d7-4521-905c-aeaa1d4e0ca4: !Template + answer_choices: null + id: 891fa5eb-33d7-4521-905c-aeaa1d4e0ca4 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' + 8aca02d8-ce15-4698-96d7-afd2ba792cf8: !Template + answer_choices: null + id: 8aca02d8-ce15-4698-96d7-afd2ba792cf8 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + 9e9ffa6b-0938-450f-877c-8d68f36b34d4: !Template + answer_choices: null + id: 9e9ffa6b-0938-450f-877c-8d68f36b34d4 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + b36630d8-9c80-416b-a648-8eb47c897c0f: !Template + answer_choices: null + id: b36630d8-9c80-416b-a648-8eb47c897c0f + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + bdcb8bc3-c756-4db9-9f60-f810efc151ac: !Template + answer_choices: null + id: bdcb8bc3-c756-4db9-9f60-f810efc151ac + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + d8bc3227-8fb9-4cb3-b252-257dcc1247fa: !Template + answer_choices: null + id: d8bc3227-8fb9-4cb3-b252-257dcc1247fa + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e74c996f-a680-4e8e-930b-d5b7fa2e0570: !Template + answer_choices: null + id: e74c996f-a680-4e8e-930b-d5b7fa2e0570 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' + f2650a80-740f-4f71-b733-ee4653d906df: !Template + answer_choices: null + id: f2650a80-740f-4f71-b733-ee4653d906df + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + f6ab5cfa-db83-4452-8ef6-f6ddc1391a2d: !Template + answer_choices: null + id: f6ab5cfa-db83-4452-8ef6-f6ddc1391a2d + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' diff --git a/promptsource/templates/wmt17/tr-en/templates.yaml b/promptsource/templates/wmt17/tr-en/templates.yaml new file mode 100644 index 000000000..9d13647dd --- /dev/null +++ b/promptsource/templates/wmt17/tr-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: tr-en +templates: + 0698dd68-dd49-4466-aa1b-ce5af9299a33: !Template + answer_choices: null + id: 0698dd68-dd49-4466-aa1b-ce5af9299a33 + jinja: 'Translate into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-target + reference: '' + 0835de7e-0340-4606-99af-465918e70b63: !Template + answer_choices: null + id: 0835de7e-0340-4606-99af-465918e70b63 + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-src+target + reference: '' + 114e79e2-fc02-4af2-a360-10f060b55b73: !Template + answer_choices: null + id: 114e79e2-fc02-4af2-a360-10f060b55b73 + jinja: 'Translate into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-target + reference: '' + 2784a23d-f6e5-466a-8514-1cc05faf45cf: !Template + answer_choices: null + id: 2784a23d-f6e5-466a-8514-1cc05faf45cf + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-source+target + reference: '' + 312a85f7-cd7f-4a8e-bb9a-1adcd3577799: !Template + answer_choices: null + id: 312a85f7-cd7f-4a8e-bb9a-1adcd3577799 + jinja: '{{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-target + reference: '' + 3158445b-99a2-4e94-8c59-7267865b6b7c: !Template + answer_choices: null + id: 3158445b-99a2-4e94-8c59-7267865b6b7c + jinja: 'How do you say {{translation["tr"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-tr-en-target + reference: '' + 348d9ba5-a8c7-4dd8-b6a9-521b624ea926: !Template + answer_choices: null + id: 348d9ba5-a8c7-4dd8-b6a9-521b624ea926 + jinja: ' {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-target + reference: '' + 44706451-c997-472e-98a1-679b3385dd86: !Template + answer_choices: null + id: 44706451-c997-472e-98a1-679b3385dd86 + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-target + reference: '' + 4b7c0209-2f7e-42e6-a650-32525a89005e: !Template + answer_choices: null + id: 4b7c0209-2f7e-42e6-a650-32525a89005e + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-target + reference: '' + 54f7e78c-c0c7-4915-933f-0d5fc7421841: !Template + answer_choices: null + id: 54f7e78c-c0c7-4915-933f-0d5fc7421841 + jinja: 'English: {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-source+target + reference: '' + 6c5ca811-0f1e-4e88-a3ce-5440e402d7dc: !Template + answer_choices: null + id: 6c5ca811-0f1e-4e88-a3ce-5440e402d7dc + jinja: 'Turkish: {{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-source+target + reference: '' + 72bac994-4b07-4b1f-9745-a9dcd63e8fab: !Template + answer_choices: null + id: 72bac994-4b07-4b1f-9745-a9dcd63e8fab + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-target + reference: '' + 77051969-f618-442d-90eb-705623f5857e: !Template + answer_choices: null + id: 77051969-f618-442d-90eb-705623f5857e + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-src+target + reference: '' + 7943be2e-958b-4f14-864e-25c43ce4729e: !Template + answer_choices: null + id: 7943be2e-958b-4f14-864e-25c43ce4729e + jinja: 'What is the Turkish translation of: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-target + reference: '' + a77a7de7-ac1a-4245-b52f-171edc9ec09c: !Template + answer_choices: null + id: a77a7de7-ac1a-4245-b52f-171edc9ec09c + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-source+target + reference: '' + adfe64cd-ae02-4932-9ebb-0b59b5501edb: !Template + answer_choices: null + id: adfe64cd-ae02-4932-9ebb-0b59b5501edb + jinja: 'How do you say {{translation["en"]}} in Turkish? + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-tr-target + reference: '' + c6f89166-46ed-4b58-bf48-7a2f7ba95ed3: !Template + answer_choices: null + id: c6f89166-46ed-4b58-bf48-7a2f7ba95ed3 + jinja: 'Translate English into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-src+starget + reference: '' + d0775d4e-9346-466e-a02c-7da052b2c5a3: !Template + answer_choices: null + id: d0775d4e-9346-466e-a02c-7da052b2c5a3 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-target + reference: '' + d1f699a0-5f70-4bf6-8aa5-53d3b3e883d3: !Template + answer_choices: null + id: d1f699a0-5f70-4bf6-8aa5-53d3b3e883d3 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-source+target + reference: '' + d2d816c6-4c8e-4bbf-8c54-68594c9a317d: !Template + answer_choices: null + id: d2d816c6-4c8e-4bbf-8c54-68594c9a317d + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-source+target + reference: '' + ee687d87-9133-4921-900d-809a9060e1f7: !Template + answer_choices: null + id: ee687d87-9133-4921-900d-809a9060e1f7 + jinja: 'What is the English translation of : {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-target + reference: '' + fa047d18-35d3-4c75-b7ce-7b1078a9889c: !Template + answer_choices: null + id: fa047d18-35d3-4c75-b7ce-7b1078a9889c + jinja: 'Translate Turkish into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf diff --git a/promptsource/templates/wmt17/zh-en/templates.yaml b/promptsource/templates/wmt17/zh-en/templates.yaml new file mode 100644 index 000000000..3fa2bb6f2 --- /dev/null +++ b/promptsource/templates/wmt17/zh-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt17 +subset: zh-en +templates: + 052514b1-99d4-47f5-949f-df28d0dd5590: !Template + answer_choices: null + id: 052514b1-99d4-47f5-949f-df28d0dd5590 + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-source+target + reference: '' + 0616df46-109a-4eb8-bd1a-075f0b85a145: !Template + answer_choices: null + id: 0616df46-109a-4eb8-bd1a-075f0b85a145 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-src+target + reference: '' + 0f64087a-c73c-47a2-8380-6222bc75dfd0: !Template + answer_choices: null + id: 0f64087a-c73c-47a2-8380-6222bc75dfd0 + jinja: 'Translate Chinese into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1a8e989a-f17c-4efb-8669-f175eb5c25a5: !Template + answer_choices: null + id: 1a8e989a-f17c-4efb-8669-f175eb5c25a5 + jinja: 'How do you say {{translation["en"]}} in Chinese? + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-zh-target + reference: '' + 1d82056d-254f-4df1-9b4b-332ce48faf9e: !Template + answer_choices: null + id: 1d82056d-254f-4df1-9b4b-332ce48faf9e + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-target + reference: '' + 2fe1c241-cdaf-4ff4-a7ac-78234251efba: !Template + answer_choices: null + id: 2fe1c241-cdaf-4ff4-a7ac-78234251efba + jinja: 'Chinese: {{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-source+target + reference: '' + 39c53426-27f6-45f4-8e51-268df51ba1e0: !Template + answer_choices: null + id: 39c53426-27f6-45f4-8e51-268df51ba1e0 + jinja: 'How do you say {{translation["zh"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-zh-en-target + reference: '' + 49f79048-3ff5-405d-b4ef-9897db228826: !Template + answer_choices: null + id: 49f79048-3ff5-405d-b4ef-9897db228826 + jinja: 'Translate into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-target + reference: '' + 4aea2385-5165-4df7-90bc-49a1e1affb70: !Template + answer_choices: null + id: 4aea2385-5165-4df7-90bc-49a1e1affb70 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-source+target + reference: '' + 5cb6b5fb-ae44-4c4b-a59f-be09ca4232a8: !Template + answer_choices: null + id: 5cb6b5fb-ae44-4c4b-a59f-be09ca4232a8 + jinja: ' {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-target + reference: '' + 7735cfcf-592f-4451-be38-982880c17632: !Template + answer_choices: null + id: 7735cfcf-592f-4451-be38-982880c17632 + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-target + reference: '' + 97528e7b-a094-406e-9d51-42a073770288: !Template + answer_choices: null + id: 97528e7b-a094-406e-9d51-42a073770288 + jinja: 'What is the Chinese translation of: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-target + reference: '' + 9f156de8-375c-4b23-837e-0ca5d1f8a034: !Template + answer_choices: null + id: 9f156de8-375c-4b23-837e-0ca5d1f8a034 + jinja: 'Translate English into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-src+starget + reference: '' + ac364ea4-f761-4951-b971-e2b1766f4e94: !Template + answer_choices: null + id: ac364ea4-f761-4951-b971-e2b1766f4e94 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-source+target + reference: '' + b455bed8-c1a6-4258-97b3-081c5f4c50f5: !Template + answer_choices: null + id: b455bed8-c1a6-4258-97b3-081c5f4c50f5 + jinja: 'Translate into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-target + reference: '' + bc317101-44e4-49fe-8f0f-07bee638a87a: !Template + answer_choices: null + id: bc317101-44e4-49fe-8f0f-07bee638a87a + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-target + reference: '' + bede4c7b-a660-4d33-a5d3-c6fb29bd5d0d: !Template + answer_choices: null + id: bede4c7b-a660-4d33-a5d3-c6fb29bd5d0d + jinja: 'English: {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-source+target + reference: '' + c0d710c2-3b3b-4c36-93b3-e8cedbb64748: !Template + answer_choices: null + id: c0d710c2-3b3b-4c36-93b3-e8cedbb64748 + jinja: '{{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-target + reference: '' + c7dcaeec-6c2e-4e8b-9fd2-fe145666777c: !Template + answer_choices: null + id: c7dcaeec-6c2e-4e8b-9fd2-fe145666777c + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-target + reference: '' + cb4fe4e3-0e94-4563-aae9-846c0d170104: !Template + answer_choices: null + id: cb4fe4e3-0e94-4563-aae9-846c0d170104 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-src+target + reference: '' + e18a0dea-d7d1-4f77-a5a5-c27427d447ac: !Template + answer_choices: null + id: e18a0dea-d7d1-4f77-a5a5-c27427d447ac + jinja: 'What is the English translation of : {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-target + reference: '' + e571c8a2-57e1-45eb-84cb-8f31b6a1dfc6: !Template + answer_choices: null + id: e571c8a2-57e1-45eb-84cb-8f31b6a1dfc6 + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-source+target + reference: '' diff --git a/promptsource/templates/wmt18/cs-en/templates.yaml b/promptsource/templates/wmt18/cs-en/templates.yaml new file mode 100644 index 000000000..52cc86bf3 --- /dev/null +++ b/promptsource/templates/wmt18/cs-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: cs-en +templates: + 040c8e39-deac-477f-a331-34630d0445ed: !Template + answer_choices: null + id: 040c8e39-deac-477f-a331-34630d0445ed + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target + reference: '' + 04d83e59-cd90-471d-a47c-c28d8b5c4a3b: !Template + answer_choices: null + id: 04d83e59-cd90-471d-a47c-c28d8b5c4a3b + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-cs-target + reference: '' + 061c1fe4-f3a4-4f2b-abd8-c9a3fb3024e1: !Template + answer_choices: null + id: 061c1fe4-f3a4-4f2b-abd8-c9a3fb3024e1 + jinja: 'Translate English into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-src+starget + reference: '' + 087fece4-a590-4668-b277-151e4106fc2e: !Template + answer_choices: null + id: 087fece4-a590-4668-b277-151e4106fc2e + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-target + reference: '' + 0a422243-290c-438a-84ec-970437fcaa1d: !Template + answer_choices: null + id: 0a422243-290c-438a-84ec-970437fcaa1d + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + 129856c6-778e-42a0-8b37-961952c9568a: !Template + answer_choices: null + id: 129856c6-778e-42a0-8b37-961952c9568a + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-cs-target + reference: '' + 1e43b70b-74b3-412d-b781-eab33e1f3539: !Template + answer_choices: null + id: 1e43b70b-74b3-412d-b781-eab33e1f3539 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-src+target + reference: '' + 1e92cbf9-f100-4af1-813e-28e04a5943e5: !Template + answer_choices: null + id: 1e92cbf9-f100-4af1-813e-28e04a5943e5 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + 21900f52-b821-4fce-8476-6ab56c2d5e7f: !Template + answer_choices: null + id: 21900f52-b821-4fce-8476-6ab56c2d5e7f + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-source+target + reference: '' + 424b2603-bb66-4c35-8923-5bfe1a60a4a2: !Template + answer_choices: null + id: 424b2603-bb66-4c35-8923-5bfe1a60a4a2 + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target + reference: '' + 5bb157ca-2812-4fad-bdb9-a44aaf4ae51f: !Template + answer_choices: null + id: 5bb157ca-2812-4fad-bdb9-a44aaf4ae51f + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: '' + 5ed29a0a-5deb-4506-ae64-34a77f2d338a: !Template + answer_choices: null + id: 5ed29a0a-5deb-4506-ae64-34a77f2d338a + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target + reference: '' + 617e0505-47e2-479e-ad44-81c1a0aa1c6a: !Template + answer_choices: null + id: 617e0505-47e2-479e-ad44-81c1a0aa1c6a + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target + reference: '' + 633165e3-840e-4436-ac11-0045d0eb7a52: !Template + answer_choices: null + id: 633165e3-840e-4436-ac11-0045d0eb7a52 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + 731fefa1-3b89-4e01-8aec-b83e5a056366: !Template + answer_choices: null + id: 731fefa1-3b89-4e01-8aec-b83e5a056366 + jinja: 'English: {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-source+target + reference: '' + 7bda53a6-fc4d-4252-b747-f2a875de5768: !Template + answer_choices: null + id: 7bda53a6-fc4d-4252-b747-f2a875de5768 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-src+target + reference: '' + 7ff11297-f0b7-4539-98a5-c1da068a9f35: !Template + answer_choices: null + id: 7ff11297-f0b7-4539-98a5-c1da068a9f35 + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target + reference: '' + 93aa6697-ca14-4945-bfe8-312d75d34693: !Template + answer_choices: null + id: 93aa6697-ca14-4945-bfe8-312d75d34693 + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + b9851c1d-6c3c-44be-ab35-36a81d1e0a28: !Template + answer_choices: null + id: b9851c1d-6c3c-44be-ab35-36a81d1e0a28 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-cs-source+target + reference: '' + bb1f8b1b-4bcf-4de4-a7bc-d90c7a69c6d8: !Template + answer_choices: null + id: bb1f8b1b-4bcf-4de4-a7bc-d90c7a69c6d8 + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target + reference: '' + cf9d7265-5b1d-454f-9c31-9cdc19c65fb0: !Template + answer_choices: null + id: cf9d7265-5b1d-454f-9c31-9cdc19c65fb0 + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-target + reference: '' + fa4118ef-a7ec-4a2e-8176-b4c28d67b7b7: !Template + answer_choices: null + id: fa4118ef-a7ec-4a2e-8176-b4c28d67b7b7 + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' diff --git a/promptsource/templates/wmt18/de-en/templates.yaml b/promptsource/templates/wmt18/de-en/templates.yaml new file mode 100644 index 000000000..82452fcc2 --- /dev/null +++ b/promptsource/templates/wmt18/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: de-en +templates: + 067b6cc5-c06c-4069-be26-6c4ba2acec6a: !Template + answer_choices: null + id: 067b6cc5-c06c-4069-be26-6c4ba2acec6a + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + 194a01fb-e73c-46a8-876f-c5e7a31bc306: !Template + answer_choices: null + id: 194a01fb-e73c-46a8-876f-c5e7a31bc306 + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + 22da89ee-3174-4828-b453-1c4d8e6c1542: !Template + answer_choices: null + id: 22da89ee-3174-4828-b453-1c4d8e6c1542 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 266a6865-d5e2-4400-a480-2920fc8a68df: !Template + answer_choices: null + id: 266a6865-d5e2-4400-a480-2920fc8a68df + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + 2c9e97db-2782-4afd-9a91-abb7e1477388: !Template + answer_choices: null + id: 2c9e97db-2782-4afd-9a91-abb7e1477388 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 3fa7947d-3e69-48c6-a1ad-780ab6a50e9f: !Template + answer_choices: null + id: 3fa7947d-3e69-48c6-a1ad-780ab6a50e9f + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + 5f25db3a-af08-4c3f-9aa0-6c88995298de: !Template + answer_choices: null + id: 5f25db3a-af08-4c3f-9aa0-6c88995298de + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 6112b376-66fd-482b-a57f-7395c84f00ad: !Template + answer_choices: null + id: 6112b376-66fd-482b-a57f-7395c84f00ad + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + 6fa7b2bc-b98b-4d4e-af60-ff18619575fa: !Template + answer_choices: null + id: 6fa7b2bc-b98b-4d4e-af60-ff18619575fa + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + 7b5a9486-be04-4c3e-a623-5c7aea520727: !Template + answer_choices: null + id: 7b5a9486-be04-4c3e-a623-5c7aea520727 + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + 88756e5a-f9e4-4beb-9c12-cfb8d0b0502a: !Template + answer_choices: null + id: 88756e5a-f9e4-4beb-9c12-cfb8d0b0502a + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + 892bc062-8275-4e7a-a4a6-40ec9ce917a5: !Template + answer_choices: null + id: 892bc062-8275-4e7a-a4a6-40ec9ce917a5 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 89ea634c-a8c6-4a7a-8430-00dbcf70b94d: !Template + answer_choices: null + id: 89ea634c-a8c6-4a7a-8430-00dbcf70b94d + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' + 8a93706e-13dc-4e8c-a9bb-0a82567d650d: !Template + answer_choices: null + id: 8a93706e-13dc-4e8c-a9bb-0a82567d650d + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + 94b1f824-97d2-4925-9f4f-e678b35e397d: !Template + answer_choices: null + id: 94b1f824-97d2-4925-9f4f-e678b35e397d + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + af129188-cd5f-4fdb-bb3f-cbdd8c8c215e: !Template + answer_choices: null + id: af129188-cd5f-4fdb-bb3f-cbdd8c8c215e + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + baf0d723-4dc0-4d53-a8d7-0123c524ae33: !Template + answer_choices: null + id: baf0d723-4dc0-4d53-a8d7-0123c524ae33 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + bdd0c786-1d66-4213-a5b4-f5fb60ec118a: !Template + answer_choices: null + id: bdd0c786-1d66-4213-a5b4-f5fb60ec118a + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + d1f5d99e-c50e-4a05-961d-c6d20a0a689f: !Template + answer_choices: null + id: d1f5d99e-c50e-4a05-961d-c6d20a0a689f + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + e2922fed-8ebb-4bf5-a538-7b406fab6739: !Template + answer_choices: null + id: e2922fed-8ebb-4bf5-a538-7b406fab6739 + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' + e49a095f-1d19-4dc2-ab79-dd971c460bdb: !Template + answer_choices: null + id: e49a095f-1d19-4dc2-ab79-dd971c460bdb + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' + fc78b44f-a4fb-4d18-a9a8-cf3eff9f9b27: !Template + answer_choices: null + id: fc78b44f-a4fb-4d18-a9a8-cf3eff9f9b27 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' diff --git a/promptsource/templates/wmt18/et-en/templates.yaml b/promptsource/templates/wmt18/et-en/templates.yaml new file mode 100644 index 000000000..962d500c7 --- /dev/null +++ b/promptsource/templates/wmt18/et-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: et-en +templates: + 181992c7-17cc-4be2-9071-21e72add55bb: !Template + answer_choices: null + id: 181992c7-17cc-4be2-9071-21e72add55bb + jinja: 'What is the Estonian translation of: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-et-target + reference: '' + 26187d14-713a-4ffc-b8c6-9cb6c28286b2: !Template + answer_choices: null + id: 26187d14-713a-4ffc-b8c6-9cb6c28286b2 + jinja: 'If the original version says: {{translation["en"]}}; then the Estonian + version should say: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-et-target + reference: '' + 39760f5a-2458-4d34-8d98-2c59799002ea: !Template + answer_choices: null + id: 39760f5a-2458-4d34-8d98-2c59799002ea + jinja: 'Given the following passage: {{translation["en"]}} , a good Estonian translation + is: ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-et-target + reference: '' + 3f96bd3d-73fa-4b75-9013-2ece6fdbd375: !Template + answer_choices: null + id: 3f96bd3d-73fa-4b75-9013-2ece6fdbd375 + jinja: 'What is the English translation of : {{translation["et"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-et-en-target + reference: '' + 4bacbd39-c66d-4cb5-9b09-083276033257: !Template + answer_choices: null + id: 4bacbd39-c66d-4cb5-9b09-083276033257 + jinja: 'How do you say {{translation["en"]}} in Estonian? + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-et-target + reference: '' + 5e548820-5c6d-41d3-bef1-6a004798efb0: !Template + answer_choices: null + id: 5e548820-5c6d-41d3-bef1-6a004798efb0 + jinja: 'If the Estonian version says: {{translation["et"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-et-en-source+target + reference: '' + 63f48c14-f8da-4c6f-b5aa-52680a0fc260: !Template + answer_choices: null + id: 63f48c14-f8da-4c6f-b5aa-52680a0fc260 + jinja: 'How do you say {{translation["et"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-et-en-target + reference: '' + 691c6136-2efb-40b5-9645-f2f69b935ed1: !Template + answer_choices: null + id: 691c6136-2efb-40b5-9645-f2f69b935ed1 + jinja: '{{translation["et"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_et-en-target + reference: '' + 71b4588d-75f8-4df8-ade4-30f7399f0d41: !Template + answer_choices: null + id: 71b4588d-75f8-4df8-ade4-30f7399f0d41 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Estonian translation is: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-et-source+target + reference: '' + 85f4a8ae-1fbf-4bc0-b4c7-e3859e181b9a: !Template + answer_choices: null + id: 85f4a8ae-1fbf-4bc0-b4c7-e3859e181b9a + jinja: 'English: {{translation["en"]}} translates into Estonian as: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-et-source+target + reference: '' + 87e69c86-0234-4d01-ae70-98a75a86f271: !Template + answer_choices: null + id: 87e69c86-0234-4d01-ae70-98a75a86f271 + jinja: 'Given the following source text in Estonian: {{translation["et"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-et-en-source+target + reference: '' + 94caadd3-0ef3-4ad5-912c-7074a7eb7896: !Template + answer_choices: null + id: 94caadd3-0ef3-4ad5-912c-7074a7eb7896 + jinja: 'If the original version says: {{translation["et"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-et-en-target + reference: '' + 99758e66-c941-4daf-b474-bd3e0a88a940: !Template + answer_choices: null + id: 99758e66-c941-4daf-b474-bd3e0a88a940 + jinja: 'If the English version says: {{translation["en"]}}; then the Estonian + version should say: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-et-source+target + reference: '' + 9b21eeec-25e7-47ba-af98-dc52bd9f3b38: !Template + answer_choices: null + id: 9b21eeec-25e7-47ba-af98-dc52bd9f3b38 + jinja: 'Translate English into Estonian: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-et-src+starget + reference: '' + 9f64040b-b37d-43f8-af82-b09d6225af41: !Template + answer_choices: null + id: 9f64040b-b37d-43f8-af82-b09d6225af41 + jinja: 'What is the Estonian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-et-src+target + reference: '' + ae103af1-9d75-4e11-95e0-f14a1701c43a: !Template + answer_choices: null + id: ae103af1-9d75-4e11-95e0-f14a1701c43a + jinja: ' {{translation["en"]}} translates into Estonian as: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-et-target + reference: '' + b28fed6e-1099-43d0-af47-4329efa3ca10: !Template + answer_choices: null + id: b28fed6e-1099-43d0-af47-4329efa3ca10 + jinja: 'What is the English translation of the Estonian sentence: {{translation["et"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-et-en-src+target + reference: '' + b41477f3-05c1-49d6-831a-1e8518f158e0: !Template + answer_choices: null + id: b41477f3-05c1-49d6-831a-1e8518f158e0 + jinja: 'Translate Estonian into English: {{translation["et"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-et-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + b6c2be75-43eb-4fc4-b279-c8503ca90bcc: !Template + answer_choices: null + id: b6c2be75-43eb-4fc4-b279-c8503ca90bcc + jinja: 'Given the following passage: {{translation["et"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-et-en-target + reference: '' + ba2b61ec-1a12-43e3-a055-b99917fc8ab0: !Template + answer_choices: null + id: ba2b61ec-1a12-43e3-a055-b99917fc8ab0 + jinja: 'Translate into English: {{translation["et"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-et-en-target + reference: '' + dce3295f-f22f-47b3-8d6c-464b5eb4dc82: !Template + answer_choices: null + id: dce3295f-f22f-47b3-8d6c-464b5eb4dc82 + jinja: 'Translate into Estonian: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-et-target + reference: '' + eaf56de6-ee11-472f-8fa1-c9aca0025e16: !Template + answer_choices: null + id: eaf56de6-ee11-472f-8fa1-c9aca0025e16 + jinja: 'Estonian: {{translation["et"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_et-en-source+target + reference: '' diff --git a/promptsource/templates/wmt18/fi-en/templates.yaml b/promptsource/templates/wmt18/fi-en/templates.yaml new file mode 100644 index 000000000..13b312dc4 --- /dev/null +++ b/promptsource/templates/wmt18/fi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: fi-en +templates: + 0a3bcad5-23df-419a-9fe6-4743aa9db2be: !Template + answer_choices: null + id: 0a3bcad5-23df-419a-9fe6-4743aa9db2be + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-src+starget + reference: '' + 210d9576-5a70-4c26-82fc-7b1bfbe01117: !Template + answer_choices: null + id: 210d9576-5a70-4c26-82fc-7b1bfbe01117 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + 22b11d6c-e589-436b-b98a-e150d8d0efb8: !Template + answer_choices: null + id: 22b11d6c-e589-436b-b98a-e150d8d0efb8 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-target + reference: '' + 2350c901-a800-414a-b1dc-4b794aba207c: !Template + answer_choices: null + id: 2350c901-a800-414a-b1dc-4b794aba207c + jinja: 'What is the Finnish translation of: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-target + reference: '' + 30297da2-9cf8-4dc8-83c2-ad06db26b79c: !Template + answer_choices: null + id: 30297da2-9cf8-4dc8-83c2-ad06db26b79c + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-target + reference: '' + 58fdc500-1ebf-47a0-85ba-4d556f80a168: !Template + answer_choices: null + id: 58fdc500-1ebf-47a0-85ba-4d556f80a168 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-source+target + reference: '' + 6177d95f-6abe-402d-96e9-bf4a7b720fa8: !Template + answer_choices: null + id: 6177d95f-6abe-402d-96e9-bf4a7b720fa8 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-src+target + reference: '' + 8952e807-1e6a-4935-92d4-c6cd07610ddc: !Template + answer_choices: null + id: 8952e807-1e6a-4935-92d4-c6cd07610ddc + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target + reference: '' + 979d391d-cfb8-410c-ac44-6911e716aea7: !Template + answer_choices: null + id: 979d391d-cfb8-410c-ac44-6911e716aea7 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + 98a3ad8d-e40f-4804-8fea-895424a78cd1: !Template + answer_choices: null + id: 98a3ad8d-e40f-4804-8fea-895424a78cd1 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + a73902fc-d455-48b2-9ab9-3924abfe005c: !Template + answer_choices: null + id: a73902fc-d455-48b2-9ab9-3924abfe005c + jinja: 'How do you say {{translation["fi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fi-en-target + reference: '' + ad09a495-727a-4d09-b6bb-f2bf92d33c23: !Template + answer_choices: null + id: ad09a495-727a-4d09-b6bb-f2bf92d33c23 + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: '' + b1505a0a-fb50-47ca-815b-645eb93780e7: !Template + answer_choices: null + id: b1505a0a-fb50-47ca-815b-645eb93780e7 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-target + reference: '' + b59c382b-457f-4d6a-9b63-7a8caf569ab7: !Template + answer_choices: null + id: b59c382b-457f-4d6a-9b63-7a8caf569ab7 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + be8c51f5-7cc7-49b2-9cee-8aecf2abe87b: !Template + answer_choices: null + id: be8c51f5-7cc7-49b2-9cee-8aecf2abe87b + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-source+target + reference: '' + cdf46e91-2228-4809-8c06-2b2c5b4fd700: !Template + answer_choices: null + id: cdf46e91-2228-4809-8c06-2b2c5b4fd700 + jinja: 'Translate Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e4103f0f-075b-481f-81b8-f3d5a3671c86: !Template + answer_choices: null + id: e4103f0f-075b-481f-81b8-f3d5a3671c86 + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-src+target + reference: '' + ea3c2068-9385-4043-ac48-aef98ef9b26b: !Template + answer_choices: null + id: ea3c2068-9385-4043-ac48-aef98ef9b26b + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-target + reference: '' + ee44f8c4-f689-4182-908a-a47679e31b07: !Template + answer_choices: null + id: ee44f8c4-f689-4182-908a-a47679e31b07 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target + reference: '' + f0e9c69a-b0ea-46f0-9aa5-55cbbb518741: !Template + answer_choices: null + id: f0e9c69a-b0ea-46f0-9aa5-55cbbb518741 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target + reference: '' + f458a114-1e5e-43e0-819e-f91cc4a7f8a0: !Template + answer_choices: null + id: f458a114-1e5e-43e0-819e-f91cc4a7f8a0 + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target + reference: '' + fbc1a8bc-2b7c-4cdc-b79c-850dd9f1558b: !Template + answer_choices: null + id: fbc1a8bc-2b7c-4cdc-b79c-850dd9f1558b + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-source+target + reference: '' diff --git a/promptsource/templates/wmt18/ru-en/templates.yaml b/promptsource/templates/wmt18/ru-en/templates.yaml new file mode 100644 index 000000000..80b1b092d --- /dev/null +++ b/promptsource/templates/wmt18/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: ru-en +templates: + 0c24a0e1-da87-452c-a600-776a539284e4: !Template + answer_choices: null + id: 0c24a0e1-da87-452c-a600-776a539284e4 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' + 0fd27cc6-b649-447c-a855-ce73ef3eb9c3: !Template + answer_choices: null + id: 0fd27cc6-b649-447c-a855-ce73ef3eb9c3 + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + 1360a954-e86c-48cf-9345-7e658c307e15: !Template + answer_choices: null + id: 1360a954-e86c-48cf-9345-7e658c307e15 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + 20534c85-0657-4966-bf6d-7d629f463cf4: !Template + answer_choices: null + id: 20534c85-0657-4966-bf6d-7d629f463cf4 + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' + 25f15ee2-8ead-498a-9d32-586cb1207e2b: !Template + answer_choices: null + id: 25f15ee2-8ead-498a-9d32-586cb1207e2b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + 3cf75a76-473e-4297-88c5-11e0654d9b54: !Template + answer_choices: null + id: 3cf75a76-473e-4297-88c5-11e0654d9b54 + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + 589bae60-fec9-4bfe-b066-814b2a45e9c0: !Template + answer_choices: null + id: 589bae60-fec9-4bfe-b066-814b2a45e9c0 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + 5fcca54f-b5c9-4083-8a5e-5ccc1b3d945d: !Template + answer_choices: null + id: 5fcca54f-b5c9-4083-8a5e-5ccc1b3d945d + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + 6262c46e-1f29-4e2d-a3fb-1d6d88107551: !Template + answer_choices: null + id: 6262c46e-1f29-4e2d-a3fb-1d6d88107551 + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 669ee38b-d45c-46e8-8f06-5f7601f0d1ad: !Template + answer_choices: null + id: 669ee38b-d45c-46e8-8f06-5f7601f0d1ad + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 72bc2e97-139f-4e7c-a9fe-d46897341c91: !Template + answer_choices: null + id: 72bc2e97-139f-4e7c-a9fe-d46897341c91 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + 898213e4-f533-41fa-84fe-8af50182efee: !Template + answer_choices: null + id: 898213e4-f533-41fa-84fe-8af50182efee + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + a0167177-6b60-4f88-ae61-759ae406fe37: !Template + answer_choices: null + id: a0167177-6b60-4f88-ae61-759ae406fe37 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + a2058434-c434-4e31-a731-e38507384683: !Template + answer_choices: null + id: a2058434-c434-4e31-a731-e38507384683 + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + b19cb71d-b9d1-48e1-828a-b24a837be44b: !Template + answer_choices: null + id: b19cb71d-b9d1-48e1-828a-b24a837be44b + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + b2300b3c-fa83-410d-b815-246c421ae934: !Template + answer_choices: null + id: b2300b3c-fa83-410d-b815-246c421ae934 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' + b6d95e24-684f-4d12-858c-9b9bed9ddf38: !Template + answer_choices: null + id: b6d95e24-684f-4d12-858c-9b9bed9ddf38 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + c71b2bfc-4184-41d1-af8d-ebe05c8d0b21: !Template + answer_choices: null + id: c71b2bfc-4184-41d1-af8d-ebe05c8d0b21 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + ce108686-f5b1-4727-9d4b-f6225fd06863: !Template + answer_choices: null + id: ce108686-f5b1-4727-9d4b-f6225fd06863 + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' + ceb4a956-96aa-485f-bb40-d003454de404: !Template + answer_choices: null + id: ceb4a956-96aa-485f-bb40-d003454de404 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + cecb7579-8a8d-435d-8734-76b3203aebb4: !Template + answer_choices: null + id: cecb7579-8a8d-435d-8734-76b3203aebb4 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + eff4c0f6-1b4e-41c8-8f77-0efaea477eab: !Template + answer_choices: null + id: eff4c0f6-1b4e-41c8-8f77-0efaea477eab + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' diff --git a/promptsource/templates/wmt18/tr-en/templates.yaml b/promptsource/templates/wmt18/tr-en/templates.yaml new file mode 100644 index 000000000..1af67ef71 --- /dev/null +++ b/promptsource/templates/wmt18/tr-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: tr-en +templates: + 05958199-cfdc-43b3-bc2b-c85d8a676e7f: !Template + answer_choices: null + id: 05958199-cfdc-43b3-bc2b-c85d8a676e7f + jinja: '{{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-target + reference: '' + 15124e31-d05e-409b-a5b1-0a1d2c2e3e21: !Template + answer_choices: null + id: 15124e31-d05e-409b-a5b1-0a1d2c2e3e21 + jinja: 'Translate English into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-src+starget + reference: '' + 1eca1c49-9140-4663-bd9d-d071fa0f4bef: !Template + answer_choices: null + id: 1eca1c49-9140-4663-bd9d-d071fa0f4bef + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-src+target + reference: '' + 282faae6-522d-4ccc-84a8-a8a4952495af: !Template + answer_choices: null + id: 282faae6-522d-4ccc-84a8-a8a4952495af + jinja: 'Translate Turkish into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 3aec5f9b-b266-4ef6-bd0e-7f2d66de8b43: !Template + answer_choices: null + id: 3aec5f9b-b266-4ef6-bd0e-7f2d66de8b43 + jinja: 'How do you say {{translation["tr"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-tr-en-target + reference: '' + 3c763b51-7749-432a-90aa-0d53e836bbde: !Template + answer_choices: null + id: 3c763b51-7749-432a-90aa-0d53e836bbde + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-src+target + reference: '' + 3e487ca8-786c-4ef3-8f9c-17946baf1b16: !Template + answer_choices: null + id: 3e487ca8-786c-4ef3-8f9c-17946baf1b16 + jinja: 'What is the Turkish translation of: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-target + reference: '' + 4af6842b-8267-4ae0-8679-20702a954f8c: !Template + answer_choices: null + id: 4af6842b-8267-4ae0-8679-20702a954f8c + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-source+target + reference: '' + 4de63ac8-81e5-4b49-b08e-6bca4951e221: !Template + answer_choices: null + id: 4de63ac8-81e5-4b49-b08e-6bca4951e221 + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-target + reference: '' + 4edee3cc-744d-4e41-8a23-f9afabb31217: !Template + answer_choices: null + id: 4edee3cc-744d-4e41-8a23-f9afabb31217 + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-source+target + reference: '' + 616d8492-6d4f-4543-bef3-932f73b7f61e: !Template + answer_choices: null + id: 616d8492-6d4f-4543-bef3-932f73b7f61e + jinja: 'Translate into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-target + reference: '' + 744e91e0-e30c-4f1b-a2d8-35a9347b0521: !Template + answer_choices: null + id: 744e91e0-e30c-4f1b-a2d8-35a9347b0521 + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-tr-en-target + reference: '' + 75e28e26-bffe-490d-a708-900746621768: !Template + answer_choices: null + id: 75e28e26-bffe-490d-a708-900746621768 + jinja: 'What is the English translation of : {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-target + reference: '' + 8945eb29-ad14-421e-8314-89aea8c36f7a: !Template + answer_choices: null + id: 8945eb29-ad14-421e-8314-89aea8c36f7a + jinja: 'How do you say {{translation["en"]}} in Turkish? + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-tr-target + reference: '' + 8e138088-9e2b-461b-92d6-3b42c7b6e53c: !Template + answer_choices: null + id: 8e138088-9e2b-461b-92d6-3b42c7b6e53c + jinja: ' {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-target + reference: '' + 91af589e-45bb-438f-9001-776068e35f70: !Template + answer_choices: null + id: 91af589e-45bb-438f-9001-776068e35f70 + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-source+target + reference: '' + a11654bb-a98b-4439-85b0-3ec29a88d79a: !Template + answer_choices: null + id: a11654bb-a98b-4439-85b0-3ec29a88d79a + jinja: 'Turkish: {{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-source+target + reference: '' + b1313bde-5381-44c6-8b13-63775e846cf7: !Template + answer_choices: null + id: b1313bde-5381-44c6-8b13-63775e846cf7 + jinja: 'English: {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-tr-source+target + reference: '' + b4224809-6324-4251-9c95-20b95aabe603: !Template + answer_choices: null + id: b4224809-6324-4251-9c95-20b95aabe603 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-target + reference: '' + c680f15d-0d5e-4652-9fba-0f13cbfbb833: !Template + answer_choices: null + id: c680f15d-0d5e-4652-9fba-0f13cbfbb833 + jinja: 'Translate into Turkish: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-tr-target + reference: '' + f50351f7-0d4d-485a-8ab3-caa2d0735b31: !Template + answer_choices: null + id: f50351f7-0d4d-485a-8ab3-caa2d0735b31 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-target + reference: '' + fb7b4b9d-7346-4042-9d52-64f141ca8249: !Template + answer_choices: null + id: fb7b4b9d-7346-4042-9d52-64f141ca8249 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-tr-en-source+target + reference: '' diff --git a/promptsource/templates/wmt18/zh-en/templates.yaml b/promptsource/templates/wmt18/zh-en/templates.yaml new file mode 100644 index 000000000..5e5d13443 --- /dev/null +++ b/promptsource/templates/wmt18/zh-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt18 +subset: zh-en +templates: + 0df95f1d-eac4-4a12-a03e-94836f4e5ecc: !Template + answer_choices: null + id: 0df95f1d-eac4-4a12-a03e-94836f4e5ecc + jinja: 'English: {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-source+target + reference: '' + 199b7514-847e-49e4-82c9-1ef4bf38ed26: !Template + answer_choices: null + id: 199b7514-847e-49e4-82c9-1ef4bf38ed26 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-source+target + reference: '' + 19f5397e-de26-4234-a253-99f959d60bdf: !Template + answer_choices: null + id: 19f5397e-de26-4234-a253-99f959d60bdf + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-source+target + reference: '' + 3f16f2a5-1b14-4fea-ac9d-b9ec7917f2fe: !Template + answer_choices: null + id: 3f16f2a5-1b14-4fea-ac9d-b9ec7917f2fe + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-source+target + reference: '' + 440f28fa-da51-4599-a014-3cdc6a3271d0: !Template + answer_choices: null + id: 440f28fa-da51-4599-a014-3cdc6a3271d0 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-target + reference: '' + 4858f6d2-e65e-4ffb-88da-2d299d935b56: !Template + answer_choices: null + id: 4858f6d2-e65e-4ffb-88da-2d299d935b56 + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-target + reference: '' + 49e8493b-084b-4994-a028-f423df05e95c: !Template + answer_choices: null + id: 49e8493b-084b-4994-a028-f423df05e95c + jinja: 'What is the English translation of : {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-target + reference: '' + 58cf0580-c83f-431d-b60c-d982285110a4: !Template + answer_choices: null + id: 58cf0580-c83f-431d-b60c-d982285110a4 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-src+target + reference: '' + 66c75f62-572e-4fb3-84f2-0c8ee6b7285b: !Template + answer_choices: null + id: 66c75f62-572e-4fb3-84f2-0c8ee6b7285b + jinja: '{{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-target + reference: '' + 79800cec-9256-41b5-855f-8a7075f4287e: !Template + answer_choices: null + id: 79800cec-9256-41b5-855f-8a7075f4287e + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-source+target + reference: '' + 7f159861-f50f-46b7-b806-9b77e5b6a13d: !Template + answer_choices: null + id: 7f159861-f50f-46b7-b806-9b77e5b6a13d + jinja: 'Translate Chinese into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 7ffd2f94-f31c-426a-b8f2-a6f98a82a3c8: !Template + answer_choices: null + id: 7ffd2f94-f31c-426a-b8f2-a6f98a82a3c8 + jinja: 'How do you say {{translation["zh"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-zh-en-target + reference: '' + 962010d1-7052-462e-a5f0-16bd18d8673f: !Template + answer_choices: null + id: 962010d1-7052-462e-a5f0-16bd18d8673f + jinja: ' {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-target + reference: '' + aba5c7bc-1bbe-4cbb-927c-d6184b05580a: !Template + answer_choices: null + id: aba5c7bc-1bbe-4cbb-927c-d6184b05580a + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-target + reference: '' + aec5b3d3-5787-47cd-b2df-41a4b6a20219: !Template + answer_choices: null + id: aec5b3d3-5787-47cd-b2df-41a4b6a20219 + jinja: 'Translate into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-target + reference: '' + c5689f05-ab0d-4e77-a712-4bb223dd29db: !Template + answer_choices: null + id: c5689f05-ab0d-4e77-a712-4bb223dd29db + jinja: 'Translate English into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-src+starget + reference: '' + c82cb77c-0333-4dab-8472-f8864fd24148: !Template + answer_choices: null + id: c82cb77c-0333-4dab-8472-f8864fd24148 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-src+target + reference: '' + cb635609-34c5-40c4-9464-9d66fcafb84c: !Template + answer_choices: null + id: cb635609-34c5-40c4-9464-9d66fcafb84c + jinja: 'Translate into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-target + reference: '' + df192cf1-c074-4de6-ab60-6dd6c22ba4fc: !Template + answer_choices: null + id: df192cf1-c074-4de6-ab60-6dd6c22ba4fc + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-target + reference: '' + e435476b-c028-4e7b-a7ff-bd1bf02a4c7b: !Template + answer_choices: null + id: e435476b-c028-4e7b-a7ff-bd1bf02a4c7b + jinja: 'How do you say {{translation["en"]}} in Chinese? + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-zh-target + reference: '' + e6836792-5d2c-4a19-aefa-31f61a5e3d48: !Template + answer_choices: null + id: e6836792-5d2c-4a19-aefa-31f61a5e3d48 + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-source+target + reference: '' + f736430b-f0c8-47bd-b261-30225b6ca1f3: !Template + answer_choices: null + id: f736430b-f0c8-47bd-b261-30225b6ca1f3 + jinja: 'What is the Chinese translation of: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-target + reference: '' diff --git a/promptsource/templates/wmt19/de-en/templates.yaml b/promptsource/templates/wmt19/de-en/templates.yaml new file mode 100644 index 000000000..828333890 --- /dev/null +++ b/promptsource/templates/wmt19/de-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: de-en +templates: + 12c560a2-d578-4585-8b1c-c2d2dde494d0: !Template + answer_choices: null + id: 12c560a2-d578-4585-8b1c-c2d2dde494d0 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-target + reference: '' + 16a88c9b-7408-415e-9acd-955725d46d11: !Template + answer_choices: null + id: 16a88c9b-7408-415e-9acd-955725d46d11 + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' + 1a0b5fa3-51e7-42d3-a3c3-e9f54bae86d0: !Template + answer_choices: null + id: 1a0b5fa3-51e7-42d3-a3c3-e9f54bae86d0 + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-target + reference: '' + 22b45127-3245-4563-82fc-259ac56b1299: !Template + answer_choices: null + id: 22b45127-3245-4563-82fc-259ac56b1299 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 39ffd410-aa87-4ea4-b9c1-f275fa819e40: !Template + answer_choices: null + id: 39ffd410-aa87-4ea4-b9c1-f275fa819e40 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + 3a825c7b-b1c9-410b-9f2c-bd03e17cc8aa: !Template + answer_choices: null + id: 3a825c7b-b1c9-410b-9f2c-bd03e17cc8aa + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 4503c426-c5f0-496a-a6c6-3bfb82a6d9b8: !Template + answer_choices: null + id: 4503c426-c5f0-496a-a6c6-3bfb82a6d9b8 + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: '' + 507d95a9-1866-4af3-ae4a-1c59a18859b6: !Template + answer_choices: null + id: 507d95a9-1866-4af3-ae4a-1c59a18859b6 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + 57cda6d4-aa45-4410-b44e-fa4379a285fc: !Template + answer_choices: null + id: 57cda6d4-aa45-4410-b44e-fa4379a285fc + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 7850c160-ac89-4d1c-b888-344f3a897a6c: !Template + answer_choices: null + id: 7850c160-ac89-4d1c-b888-344f3a897a6c + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-target + reference: '' + 86406e46-d972-4acb-95ba-416a0fd797ff: !Template + answer_choices: null + id: 86406e46-d972-4acb-95ba-416a0fd797ff + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 8d5d8e4c-6a3e-4628-a665-58876376514b: !Template + answer_choices: null + id: 8d5d8e4c-6a3e-4628-a665-58876376514b + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-src+target + reference: '' + b969ff79-9878-4458-9bf0-1a65059c7e6d: !Template + answer_choices: null + id: b969ff79-9878-4458-9bf0-1a65059c7e6d + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-target + reference: '' + bfe56bc0-5579-4f9e-87b4-9b6f9e4dca40: !Template + answer_choices: null + id: bfe56bc0-5579-4f9e-87b4-9b6f9e4dca40 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + c8d35e0b-e1fd-4d3e-b85b-41146355ea93: !Template + answer_choices: null + id: c8d35e0b-e1fd-4d3e-b85b-41146355ea93 + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-en-target + reference: '' + cac26ba0-a319-4040-bda7-e3c40f354d66: !Template + answer_choices: null + id: cac26ba0-a319-4040-bda7-e3c40f354d66 + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-target + reference: '' + cc09b2e4-7903-427c-8cc4-23767ea2c0a7: !Template + answer_choices: null + id: cc09b2e4-7903-427c-8cc4-23767ea2c0a7 + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + cdbb3f7f-fe3e-4498-9cad-26ab67155889: !Template + answer_choices: null + id: cdbb3f7f-fe3e-4498-9cad-26ab67155889 + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target + reference: '' + d950dcab-7731-4f89-8447-9566a1ffeb9f: !Template + answer_choices: null + id: d950dcab-7731-4f89-8447-9566a1ffeb9f + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: '' + dbc67f6b-a88d-4a4f-9977-ecee61cd03ec: !Template + answer_choices: null + id: dbc67f6b-a88d-4a4f-9977-ecee61cd03ec + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + ea7609bc-c4b5-4c54-865d-35502fffeaa5: !Template + answer_choices: null + id: ea7609bc-c4b5-4c54-865d-35502fffeaa5 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-src+target + reference: '' + f4dd1aef-4890-43a9-a860-ec1d3f915f3f: !Template + answer_choices: null + id: f4dd1aef-4890-43a9-a860-ec1d3f915f3f + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-src+starget + reference: '' diff --git a/promptsource/templates/wmt19/fi-en/templates.yaml b/promptsource/templates/wmt19/fi-en/templates.yaml new file mode 100644 index 000000000..93f7e2a4e --- /dev/null +++ b/promptsource/templates/wmt19/fi-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: fi-en +templates: + 1001cedb-97cb-47ad-8e7b-d50b03968dbc: !Template + answer_choices: null + id: 1001cedb-97cb-47ad-8e7b-d50b03968dbc + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-src+target + reference: '' + 10fa4a51-7d55-45a3-8cd7-7cba89f3b975: !Template + answer_choices: null + id: 10fa4a51-7d55-45a3-8cd7-7cba89f3b975 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-target + reference: '' + 16d5b112-ec8c-4fe1-9352-472ec5681314: !Template + answer_choices: null + id: 16d5b112-ec8c-4fe1-9352-472ec5681314 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + 1b5130e3-4630-41ad-9319-b3810fc02017: !Template + answer_choices: null + id: 1b5130e3-4630-41ad-9319-b3810fc02017 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-target + reference: '' + 2db9d5f4-456c-4dd0-8550-abe67de938de: !Template + answer_choices: null + id: 2db9d5f4-456c-4dd0-8550-abe67de938de + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fi-source+target + reference: '' + 339f88b7-6f8a-4b9a-8a29-85dfcc6d6dc4: !Template + answer_choices: null + id: 339f88b7-6f8a-4b9a-8a29-85dfcc6d6dc4 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + 3e010328-0008-4452-b00d-1fee5f13f5db: !Template + answer_choices: null + id: 3e010328-0008-4452-b00d-1fee5f13f5db + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target + reference: '' + 58713529-83aa-4b04-8baf-2f2c0df4de01: !Template + answer_choices: null + id: 58713529-83aa-4b04-8baf-2f2c0df4de01 + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-target + reference: '' + 5d2e040b-4a63-4192-aa57-e4762208b9c4: !Template + answer_choices: null + id: 5d2e040b-4a63-4192-aa57-e4762208b9c4 + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target + reference: '' + 61213da3-947e-452a-b693-bec694d4cdc0: !Template + answer_choices: null + id: 61213da3-947e-452a-b693-bec694d4cdc0 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-target + reference: '' + 6568df70-fb26-481d-a87b-52840af9870b: !Template + answer_choices: null + id: 6568df70-fb26-481d-a87b-52840af9870b + jinja: 'Translate Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 761c8d1e-bba4-4b1e-8289-a61a7002bfb7: !Template + answer_choices: null + id: 761c8d1e-bba4-4b1e-8289-a61a7002bfb7 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target + reference: '' + 782d11f2-5abf-43ef-aa4b-63bd537626fb: !Template + answer_choices: null + id: 782d11f2-5abf-43ef-aa4b-63bd537626fb + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: '' + 83535f54-e2c9-4048-81b8-44d815519021: !Template + answer_choices: null + id: 83535f54-e2c9-4048-81b8-44d815519021 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + 9b351621-e584-4f25-9aa1-2bd44296f166: !Template + answer_choices: null + id: 9b351621-e584-4f25-9aa1-2bd44296f166 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + 9f87b91c-81be-4746-b5b7-404f8ce0cd8e: !Template + answer_choices: null + id: 9f87b91c-81be-4746-b5b7-404f8ce0cd8e + jinja: 'What is the Finnish translation of: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-target + reference: '' + b8a9b65f-fb04-457d-b5ca-937fc746e7f0: !Template + answer_choices: null + id: b8a9b65f-fb04-457d-b5ca-937fc746e7f0 + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-fi-source+target + reference: '' + c8b36077-2743-4b30-b31f-609429b34dfe: !Template + answer_choices: null + id: c8b36077-2743-4b30-b31f-609429b34dfe + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fi-en-source+target + reference: '' + caa1ab64-22c3-48e7-8f23-d36e89667f01: !Template + answer_choices: null + id: caa1ab64-22c3-48e7-8f23-d36e89667f01 + jinja: 'How do you say {{translation["fi"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fi-en-target + reference: '' + cab28bdd-009c-4a17-a9d1-388a932268d8: !Template + answer_choices: null + id: cab28bdd-009c-4a17-a9d1-388a932268d8 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target + reference: '' + eb18840d-5589-4fd2-8680-64bf0cc14040: !Template + answer_choices: null + id: eb18840d-5589-4fd2-8680-64bf0cc14040 + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-src+starget + reference: '' + fd7d15e0-58d6-4aee-9623-4ceb47a2e3b5: !Template + answer_choices: null + id: fd7d15e0-58d6-4aee-9623-4ceb47a2e3b5 + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-src+target + reference: '' diff --git a/promptsource/templates/wmt19/fr-de/templates.yaml b/promptsource/templates/wmt19/fr-de/templates.yaml new file mode 100644 index 000000000..f207d4b8a --- /dev/null +++ b/promptsource/templates/wmt19/fr-de/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: fr-de +templates: + 06e5d46e-8710-43c8-bf44-c2303dd16cfd: !Template + answer_choices: null + id: 06e5d46e-8710-43c8-bf44-c2303dd16cfd + jinja: 'If the original version says: {{translation["fr"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fr-de-target + reference: '' + 0e4bda14-aa54-4dd4-8129-133f93125b80: !Template + answer_choices: null + id: 0e4bda14-aa54-4dd4-8129-133f93125b80 + jinja: 'French: {{translation["fr"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fr-de-source+target + reference: '' + 109e862f-27ea-451d-b31f-653c018845fa: !Template + answer_choices: null + id: 109e862f-27ea-451d-b31f-653c018845fa + jinja: 'If the French version says: {{translation["fr"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fr-de-source+target + reference: '' + 15ce04c0-8fca-4848-b211-bbc2911ab9a0: !Template + answer_choices: null + id: 15ce04c0-8fca-4848-b211-bbc2911ab9a0 + jinja: 'German: {{translation["de"]}} translates into French as: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-fr-source+target + reference: '' + 1e271797-2f0c-464e-bc96-b5015b4fcb06: !Template + answer_choices: null + id: 1e271797-2f0c-464e-bc96-b5015b4fcb06 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + French translation is: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-fr-source+target + reference: '' + 391bbba9-45e6-44e2-b706-612ddfd9a222: !Template + answer_choices: null + id: 391bbba9-45e6-44e2-b706-612ddfd9a222 + jinja: 'If the German version says: {{translation["de"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-fr-source+target + reference: '' + 47192a46-77ae-45f2-8be6-aff4b06d479d: !Template + answer_choices: null + id: 47192a46-77ae-45f2-8be6-aff4b06d479d + jinja: '{{translation["fr"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fr-de-target + reference: '' + 4bef3760-3594-489e-b9d7-94d9047d348c: !Template + answer_choices: null + id: 4bef3760-3594-489e-b9d7-94d9047d348c + jinja: 'Given the following passage: {{translation["de"]}} , a good French translation + is: ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-fr-target + reference: '' + 51476543-e538-482f-abe1-d334f432d879: !Template + answer_choices: null + id: 51476543-e538-482f-abe1-d334f432d879 + jinja: 'What is the French translation of: {{translation["de"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-fr-target + reference: '' + 5968a41f-823d-4c76-aead-c6b2642ed911: !Template + answer_choices: null + id: 5968a41f-823d-4c76-aead-c6b2642ed911 + jinja: 'How do you say {{translation["fr"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-fr-de-target + reference: '' + 5e7e4a8f-b7a9-4643-9eda-e0e4f264ac67: !Template + answer_choices: null + id: 5e7e4a8f-b7a9-4643-9eda-e0e4f264ac67 + jinja: 'What is the German translation of : {{translation["fr"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fr-de-target + reference: '' + 65fadf27-ecd0-4164-aa19-65d0124e32eb: !Template + answer_choices: null + id: 65fadf27-ecd0-4164-aa19-65d0124e32eb + jinja: 'Translate German into French: {{translation["de"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-fr-src+starget + reference: '' + 7ce086f0-f1cd-4ac6-9e62-c1debb631793: !Template + answer_choices: null + id: 7ce086f0-f1cd-4ac6-9e62-c1debb631793 + jinja: 'Translate French into German: {{translation["fr"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fr-de-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 7f129b9b-7f7e-4f33-a8ae-3d7141da1caa: !Template + answer_choices: null + id: 7f129b9b-7f7e-4f33-a8ae-3d7141da1caa + jinja: 'Translate into German: {{translation["fr"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fr-de-target + reference: '' + 8b31f232-6b09-4f80-b08d-990c00f571a1: !Template + answer_choices: null + id: 8b31f232-6b09-4f80-b08d-990c00f571a1 + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + German translation is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fr-de-source+target + reference: '' + a50c3409-850e-483d-aab3-db59cb467502: !Template + answer_choices: null + id: a50c3409-850e-483d-aab3-db59cb467502 + jinja: 'What is the German translation of the French sentence: {{translation["fr"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fr-de-src+target + reference: '' + c16b8dca-ca91-430b-828a-13ec0666570f: !Template + answer_choices: null + id: c16b8dca-ca91-430b-828a-13ec0666570f + jinja: 'Given the following passage: {{translation["fr"]}} , a good German translation + is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fr-de-target + reference: '' + c5cf6a4a-3d88-497a-ab2a-c261a2acc4e0: !Template + answer_choices: null + id: c5cf6a4a-3d88-497a-ab2a-c261a2acc4e0 + jinja: 'How do you say {{translation["de"]}} in French? + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-de-fr-target + reference: '' + d5560280-7e1d-4425-a65e-9e3bef881399: !Template + answer_choices: null + id: d5560280-7e1d-4425-a65e-9e3bef881399 + jinja: ' {{translation["de"]}} translates into French as: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-fr-target + reference: '' + dd78b916-da02-4d4e-8e28-2a1fc12faf53: !Template + answer_choices: null + id: dd78b916-da02-4d4e-8e28-2a1fc12faf53 + jinja: 'If the original version says: {{translation["de"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-fr-target + reference: '' + e908e99f-8eb0-46c8-86d4-8d13b654b782: !Template + answer_choices: null + id: e908e99f-8eb0-46c8-86d4-8d13b654b782 + jinja: 'What is the French translation of the German sentence: {{translation["de"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-fr-src+target + reference: '' + f37939c2-bc7b-43bc-90d4-2c368e6b3be8: !Template + answer_choices: null + id: f37939c2-bc7b-43bc-90d4-2c368e6b3be8 + jinja: 'Translate into French: {{translation["de"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-fr-target + reference: '' diff --git a/promptsource/templates/wmt19/gu-en/templates.yaml b/promptsource/templates/wmt19/gu-en/templates.yaml new file mode 100644 index 000000000..e3b2f2eb9 --- /dev/null +++ b/promptsource/templates/wmt19/gu-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: gu-en +templates: + 024097fc-b7b4-487b-bd65-427f69fc2f9f: !Template + answer_choices: null + id: 024097fc-b7b4-487b-bd65-427f69fc2f9f + jinja: 'If the English version says: {{translation["en"]}}; then the Gurajati + version should say: + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-gu-source+target + reference: '' + 1508501b-a090-4d29-aa7e-faa8fef35aa0: !Template + answer_choices: null + id: 1508501b-a090-4d29-aa7e-faa8fef35aa0 + jinja: 'Translate Gurajati into English: {{translation["gu"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-gu-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1a362864-c9a8-47f2-8e94-b1b78b7f02df: !Template + answer_choices: null + id: 1a362864-c9a8-47f2-8e94-b1b78b7f02df + jinja: 'If the Gurajati version says: {{translation["gu"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-gu-en-source+target + reference: '' + 23ce908d-804c-42d0-8ba1-abf39d614501: !Template + answer_choices: null + id: 23ce908d-804c-42d0-8ba1-abf39d614501 + jinja: 'How do you say {{translation["gu"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-gu-en-target + reference: '' + 2d883e74-d1de-4fb5-864a-0826b93aeffa: !Template + answer_choices: null + id: 2d883e74-d1de-4fb5-864a-0826b93aeffa + jinja: 'What is the Gurajati translation of the English sentence: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-gu-src+target + reference: '' + 3cc30841-6b2d-4fa7-9c0b-a1e9c276a3b0: !Template + answer_choices: null + id: 3cc30841-6b2d-4fa7-9c0b-a1e9c276a3b0 + jinja: '{{translation["gu"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_gu-en-target + reference: '' + 42b4d3e8-5dba-4b0d-8c5e-af24d69359a6: !Template + answer_choices: null + id: 42b4d3e8-5dba-4b0d-8c5e-af24d69359a6 + jinja: ' {{translation["en"]}} translates into Gurajati as: + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-gu-target + reference: '' + 493595fb-0f0e-4821-a726-e1a764e8138f: !Template + answer_choices: null + id: 493595fb-0f0e-4821-a726-e1a764e8138f + jinja: 'Translate English into Gurajati: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-gu-src+starget + reference: '' + 4a62cf8b-cff0-433d-b69c-afbfe78254e2: !Template + answer_choices: null + id: 4a62cf8b-cff0-433d-b69c-afbfe78254e2 + jinja: 'Given the following passage: {{translation["gu"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-gu-en-target + reference: '' + 4d82a1d8-ceb6-4258-a55b-b7473a3f294b: !Template + answer_choices: null + id: 4d82a1d8-ceb6-4258-a55b-b7473a3f294b + jinja: 'What is the Gurajati translation of: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-gu-target + reference: '' + 5910df5c-7738-4706-880c-a2f8e0e66f03: !Template + answer_choices: null + id: 5910df5c-7738-4706-880c-a2f8e0e66f03 + jinja: 'Gurajati: {{translation["gu"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_gu-en-source+target + reference: '' + 6531eac2-5d58-48f2-963b-d617a084d57b: !Template + answer_choices: null + id: 6531eac2-5d58-48f2-963b-d617a084d57b + jinja: 'English: {{translation["en"]}} translates into Gurajati as: + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-gu-source+target + reference: '' + 7200ee57-175b-4a62-b408-0ad05efb8a4f: !Template + answer_choices: null + id: 7200ee57-175b-4a62-b408-0ad05efb8a4f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Gurajati translation is: + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-gu-source+target + reference: '' + 78d0b841-b9dc-49fa-8546-ef5736490e69: !Template + answer_choices: null + id: 78d0b841-b9dc-49fa-8546-ef5736490e69 + jinja: 'How do you say {{translation["en"]}} in Gurajati? + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-gu-target + reference: '' + a59fa17f-6710-4a73-8e98-7926b3b1147d: !Template + answer_choices: null + id: a59fa17f-6710-4a73-8e98-7926b3b1147d + jinja: 'If the original version says: {{translation["en"]}}; then the Gurajati + version should say: + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-gu-target + reference: '' + b6f836db-a51e-4a9b-a5eb-153248a88df9: !Template + answer_choices: null + id: b6f836db-a51e-4a9b-a5eb-153248a88df9 + jinja: 'Given the following source text in Gurajati: {{translation["gu"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-gu-en-source+target + reference: '' + b8f138f2-127e-4107-aa19-51ccf59fcebf: !Template + answer_choices: null + id: b8f138f2-127e-4107-aa19-51ccf59fcebf + jinja: 'What is the English translation of : {{translation["gu"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-gu-en-target + reference: '' + bc0b95d1-dbba-42df-8c68-bd9219fd9ad6: !Template + answer_choices: null + id: bc0b95d1-dbba-42df-8c68-bd9219fd9ad6 + jinja: 'What is the English translation of the Gurajati sentence: {{translation["gu"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-gu-en-src+target + reference: '' + c5bb6887-f5be-426f-a275-a456a5ffe4dd: !Template + answer_choices: null + id: c5bb6887-f5be-426f-a275-a456a5ffe4dd + jinja: 'Translate into Gurajati: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-gu-target + reference: '' + df05bb02-ef33-4e4e-b16e-0ec9a2cb067b: !Template + answer_choices: null + id: df05bb02-ef33-4e4e-b16e-0ec9a2cb067b + jinja: 'If the original version says: {{translation["gu"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-gu-en-target + reference: '' + ed90082f-a447-4830-8842-5d8ab6d59f5c: !Template + answer_choices: null + id: ed90082f-a447-4830-8842-5d8ab6d59f5c + jinja: 'Given the following passage: {{translation["en"]}} , a good Gurajati translation + is: ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-gu-target + reference: '' + fe5a2406-1643-421f-b84d-bee3efe549b5: !Template + answer_choices: null + id: fe5a2406-1643-421f-b84d-bee3efe549b5 + jinja: 'Translate into English: {{translation["gu"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-gu-en-target + reference: '' diff --git a/promptsource/templates/wmt19/kk-en/templates.yaml b/promptsource/templates/wmt19/kk-en/templates.yaml new file mode 100644 index 000000000..53d19b26c --- /dev/null +++ b/promptsource/templates/wmt19/kk-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: kk-en +templates: + 157c74e2-4eaf-492a-b3d5-b7b55df8a8f0: !Template + answer_choices: null + id: 157c74e2-4eaf-492a-b3d5-b7b55df8a8f0 + jinja: 'How do you say {{translation["en"]}} in Kazakh? + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-kk-target + reference: '' + 175347ec-2186-4a10-8b83-f88a604c87e2: !Template + answer_choices: null + id: 175347ec-2186-4a10-8b83-f88a604c87e2 + jinja: 'If the original version says: {{translation["kk"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-kk-en-target + reference: '' + 1d967f47-7d17-4755-9399-b436f9dd3267: !Template + answer_choices: null + id: 1d967f47-7d17-4755-9399-b436f9dd3267 + jinja: 'If the original version says: {{translation["en"]}}; then the Kazakh version + should say: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-kk-target + reference: '' + 271a87d3-43cb-4962-9446-ee5df99912dc: !Template + answer_choices: null + id: 271a87d3-43cb-4962-9446-ee5df99912dc + jinja: 'What is the Kazakh translation of the English sentence: {{translation["en"]}} + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-kk-src+target + reference: '' + 38e6b64b-eb3c-4efc-aca4-7172784804fc: !Template + answer_choices: null + id: 38e6b64b-eb3c-4efc-aca4-7172784804fc + jinja: 'Given the following passage: {{translation["en"]}} , a good Kazakh translation + is: ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-kk-target + reference: '' + 43cf4166-7da6-42fd-9837-067646fbdc5a: !Template + answer_choices: null + id: 43cf4166-7da6-42fd-9837-067646fbdc5a + jinja: 'Translate Kazakh into English: {{translation["kk"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-kk-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 6c6a16ee-80d9-4b2a-a087-47b741344eb3: !Template + answer_choices: null + id: 6c6a16ee-80d9-4b2a-a087-47b741344eb3 + jinja: 'Given the following passage: {{translation["kk"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-kk-en-target + reference: '' + 70054e8a-8c17-4fe3-a89e-af22f97d0690: !Template + answer_choices: null + id: 70054e8a-8c17-4fe3-a89e-af22f97d0690 + jinja: '{{translation["kk"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_kk-en-target + reference: '' + 710acdc8-6990-4d37-a6d1-53d17d6744d6: !Template + answer_choices: null + id: 710acdc8-6990-4d37-a6d1-53d17d6744d6 + jinja: 'What is the Kazakh translation of: {{translation["en"]}} + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-kk-target + reference: '' + 74a834e6-df41-40b8-99f6-015b4285d6c6: !Template + answer_choices: null + id: 74a834e6-df41-40b8-99f6-015b4285d6c6 + jinja: 'Translate English into Kazakh: {{translation["en"]}} + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-kk-src+starget + reference: '' + 8f2e8da8-3864-4a93-8362-a82f4718f842: !Template + answer_choices: null + id: 8f2e8da8-3864-4a93-8362-a82f4718f842 + jinja: 'Kazakh: {{translation["kk"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_kk-en-source+target + reference: '' + 9158c439-cc8d-40a2-a692-693564c91db3: !Template + answer_choices: null + id: 9158c439-cc8d-40a2-a692-693564c91db3 + jinja: 'If the English version says: {{translation["en"]}}; then the Kazakh version + should say: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-kk-source+target + reference: '' + 9389ace0-7c38-445a-be50-0578249166b5: !Template + answer_choices: null + id: 9389ace0-7c38-445a-be50-0578249166b5 + jinja: 'If the Kazakh version says: {{translation["kk"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-kk-en-source+target + reference: '' + a057638b-49b5-414e-a63b-3e70f795ea5f: !Template + answer_choices: null + id: a057638b-49b5-414e-a63b-3e70f795ea5f + jinja: 'How do you say {{translation["kk"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-kk-en-target + reference: '' + afaa595f-deae-4e35-917c-d58151ec6d18: !Template + answer_choices: null + id: afaa595f-deae-4e35-917c-d58151ec6d18 + jinja: 'Translate into English: {{translation["kk"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-kk-en-target + reference: '' + b101e4db-6b02-436c-9be0-d2fee827576f: !Template + answer_choices: null + id: b101e4db-6b02-436c-9be0-d2fee827576f + jinja: 'What is the English translation of : {{translation["kk"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-kk-en-target + reference: '' + d11653e9-8fec-430e-9a97-f2195fb71c02: !Template + answer_choices: null + id: d11653e9-8fec-430e-9a97-f2195fb71c02 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Kazakh translation is: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-kk-source+target + reference: '' + d3ce23e8-fad3-4473-8087-da01fb9af040: !Template + answer_choices: null + id: d3ce23e8-fad3-4473-8087-da01fb9af040 + jinja: 'Translate into Kazakh: {{translation["en"]}} + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-kk-target + reference: '' + d6ea1138-feed-499b-91a1-328483fabc1f: !Template + answer_choices: null + id: d6ea1138-feed-499b-91a1-328483fabc1f + jinja: ' {{translation["en"]}} translates into Kazakh as: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-kk-target + reference: '' + d7e61faa-02f8-4801-9f61-c2425b8dddf8: !Template + answer_choices: null + id: d7e61faa-02f8-4801-9f61-c2425b8dddf8 + jinja: 'Given the following source text in Kazakh: {{translation["kk"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-kk-en-source+target + reference: '' + d80b19a0-d876-4d51-a479-c5a25a640113: !Template + answer_choices: null + id: d80b19a0-d876-4d51-a479-c5a25a640113 + jinja: 'What is the English translation of the Kazakh sentence: {{translation["kk"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-kk-en-src+target + reference: '' + e94d40e1-5d03-444e-a814-5158a819842d: !Template + answer_choices: null + id: e94d40e1-5d03-444e-a814-5158a819842d + jinja: 'English: {{translation["en"]}} translates into Kazakh as: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-kk-source+target + reference: '' diff --git a/promptsource/templates/wmt19/lt-en/templates.yaml b/promptsource/templates/wmt19/lt-en/templates.yaml new file mode 100644 index 000000000..400d48977 --- /dev/null +++ b/promptsource/templates/wmt19/lt-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: lt-en +templates: + 118f65c0-dd02-4abb-a66f-47d3d8799a07: !Template + answer_choices: null + id: 118f65c0-dd02-4abb-a66f-47d3d8799a07 + jinja: 'How do you say {{translation["lt"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-lt-en-target + reference: '' + 119b07b6-d337-486c-9b6b-65a571e5821e: !Template + answer_choices: null + id: 119b07b6-d337-486c-9b6b-65a571e5821e + jinja: 'Translate into Lithuanian: {{translation["en"]}} + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-lt-target + reference: '' + 11a563fa-7dc4-4902-bc8f-437f49103165: !Template + answer_choices: null + id: 11a563fa-7dc4-4902-bc8f-437f49103165 + jinja: 'If the English version says: {{translation["en"]}}; then the Lithuanian + version should say: + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-lt-source+target + reference: '' + 1d5cea12-e7f1-4eed-9568-dc0d029ae09c: !Template + answer_choices: null + id: 1d5cea12-e7f1-4eed-9568-dc0d029ae09c + jinja: ' {{translation["en"]}} translates into Lithuanian as: + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-lt-target + reference: '' + 1d7db603-5023-4363-808d-da1720fe0e6d: !Template + answer_choices: null + id: 1d7db603-5023-4363-808d-da1720fe0e6d + jinja: 'Given the following source text in Lithuanian: {{translation["lt"]}} , + a good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-lt-en-source+target + reference: '' + 4838300a-6622-44fb-a72d-44f75fe1f4fe: !Template + answer_choices: null + id: 4838300a-6622-44fb-a72d-44f75fe1f4fe + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Lithuanian translation is: + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lt-source+target + reference: '' + 4c90fb04-938c-44a4-98da-f8e049bc58a3: !Template + answer_choices: null + id: 4c90fb04-938c-44a4-98da-f8e049bc58a3 + jinja: 'What is the Lithuanian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-lt-src+target + reference: '' + 62b686d3-b82b-4399-8f8b-3f51977b0b6c: !Template + answer_choices: null + id: 62b686d3-b82b-4399-8f8b-3f51977b0b6c + jinja: 'English: {{translation["en"]}} translates into Lithuanian as: + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-lt-source+target + reference: '' + 65576b55-81ec-4761-8774-74a30eace165: !Template + answer_choices: null + id: 65576b55-81ec-4761-8774-74a30eace165 + jinja: 'What is the English translation of : {{translation["lt"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-lt-en-target + reference: '' + 66774f5f-8435-41b6-9597-706c190def94: !Template + answer_choices: null + id: 66774f5f-8435-41b6-9597-706c190def94 + jinja: 'Lithuanian: {{translation["lt"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lt-en-source+target + reference: '' + 693fc37a-a7e8-47d7-887c-008e193d9746: !Template + answer_choices: null + id: 693fc37a-a7e8-47d7-887c-008e193d9746 + jinja: 'Translate into English: {{translation["lt"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-lt-en-target + reference: '' + 7ae534d1-5beb-4b4c-923f-1b748f4cc1bc: !Template + answer_choices: null + id: 7ae534d1-5beb-4b4c-923f-1b748f4cc1bc + jinja: 'If the Lithuanian version says: {{translation["lt"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-lt-en-source+target + reference: '' + a1c2cbeb-86eb-419c-9bd9-6579a513391f: !Template + answer_choices: null + id: a1c2cbeb-86eb-419c-9bd9-6579a513391f + jinja: 'What is the Lithuanian translation of: {{translation["en"]}} + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-lt-target + reference: '' + a50239e7-e544-4a1a-a3e1-dd58647879fa: !Template + answer_choices: null + id: a50239e7-e544-4a1a-a3e1-dd58647879fa + jinja: 'Given the following passage: {{translation["lt"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-lt-en-target + reference: '' + a5bc4337-60ea-464e-8ddd-f8595f34cc4f: !Template + answer_choices: null + id: a5bc4337-60ea-464e-8ddd-f8595f34cc4f + jinja: '{{translation["lt"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lt-en-target + reference: '' + b278a391-1bc3-42df-aabd-4891106bdf18: !Template + answer_choices: null + id: b278a391-1bc3-42df-aabd-4891106bdf18 + jinja: 'How do you say {{translation["en"]}} in Lithuanian? + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-lt-target + reference: '' + c48fb5f0-4b98-4d22-b3de-b5b9d6d06a46: !Template + answer_choices: null + id: c48fb5f0-4b98-4d22-b3de-b5b9d6d06a46 + jinja: 'Translate English into Lithuanian: {{translation["en"]}} + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-lt-src+starget + reference: '' + c8f476e6-75ff-468d-9fc9-69aea45c7054: !Template + answer_choices: null + id: c8f476e6-75ff-468d-9fc9-69aea45c7054 + jinja: 'If the original version says: {{translation["en"]}}; then the Lithuanian + version should say: + + ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-lt-target + reference: '' + d9ef689f-bd41-483d-b2a6-6fa68a90dd07: !Template + answer_choices: null + id: d9ef689f-bd41-483d-b2a6-6fa68a90dd07 + jinja: 'What is the English translation of the Lithuanian sentence: {{translation["lt"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-lt-en-src+target + reference: '' + f0fee957-425b-472e-9ef0-862ccdc1dd8e: !Template + answer_choices: null + id: f0fee957-425b-472e-9ef0-862ccdc1dd8e + jinja: 'Given the following passage: {{translation["en"]}} , a good Lithuanian + translation is: ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lt-target + reference: '' + f63651e7-2ab4-424c-87ed-aa8e809b950e: !Template + answer_choices: null + id: f63651e7-2ab4-424c-87ed-aa8e809b950e + jinja: 'Translate Lithuanian into English: {{translation["lt"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-lt-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + ff80e819-392d-4656-a75c-866d74ce5d33: !Template + answer_choices: null + id: ff80e819-392d-4656-a75c-866d74ce5d33 + jinja: 'If the original version says: {{translation["lt"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-lt-en-target + reference: '' diff --git a/promptsource/templates/wmt19/ru-en/templates.yaml b/promptsource/templates/wmt19/ru-en/templates.yaml new file mode 100644 index 000000000..81ce55fdf --- /dev/null +++ b/promptsource/templates/wmt19/ru-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: ru-en +templates: + 2222898b-a719-45b2-bd77-78f9ff7f7d8f: !Template + answer_choices: null + id: 2222898b-a719-45b2-bd77-78f9ff7f7d8f + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target + reference: '' + 25e8fb3b-9612-443e-a6a9-b4793d8f0cca: !Template + answer_choices: null + id: 25e8fb3b-9612-443e-a6a9-b4793d8f0cca + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target + reference: '' + 347eb46a-4b18-4aa9-bac8-748ec9481be4: !Template + answer_choices: null + id: 347eb46a-4b18-4aa9-bac8-748ec9481be4 + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-target + reference: '' + 3ff8be77-d920-4f79-8f81-64b022744ce4: !Template + answer_choices: null + id: 3ff8be77-d920-4f79-8f81-64b022744ce4 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-source+target + reference: '' + 493e3602-1bf2-473f-8dca-37f0064dfa86: !Template + answer_choices: null + id: 493e3602-1bf2-473f-8dca-37f0064dfa86 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + 5433c9ee-ed73-4660-b9e6-d9be4f7c70e4: !Template + answer_choices: null + id: 5433c9ee-ed73-4660-b9e6-d9be4f7c70e4 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-target + reference: '' + 64ceab1f-773e-4c4d-a721-c5964d86fffe: !Template + answer_choices: null + id: 64ceab1f-773e-4c4d-a721-c5964d86fffe + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: '' + 6a134c4c-d73a-480c-bfdc-4b180c253d87: !Template + answer_choices: null + id: 6a134c4c-d73a-480c-bfdc-4b180c253d87 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + 6cef404a-7c3c-47d0-8782-ea3b74cecc68: !Template + answer_choices: null + id: 6cef404a-7c3c-47d0-8782-ea3b74cecc68 + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-target + reference: '' + 6d92111c-1aff-46d3-bec5-38a5edeebe77: !Template + answer_choices: null + id: 6d92111c-1aff-46d3-bec5-38a5edeebe77 + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-src+starget + reference: '' + 738d2f5f-1e6a-4cae-814e-cef6368d37ca: !Template + answer_choices: null + id: 738d2f5f-1e6a-4cae-814e-cef6368d37ca + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-ru-en-target + reference: '' + 76d1e5dc-ed29-4c66-83a9-1627e68ac0b8: !Template + answer_choices: null + id: 76d1e5dc-ed29-4c66-83a9-1627e68ac0b8 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target + reference: '' + 8a8b26ec-5996-460a-826d-c87b1a38def3: !Template + answer_choices: null + id: 8a8b26ec-5996-460a-826d-c87b1a38def3 + jinja: 'Translate into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-ru-target + reference: '' + 8ab2840e-4df0-4cd5-9ac4-21c8d84af78b: !Template + answer_choices: null + id: 8ab2840e-4df0-4cd5-9ac4-21c8d84af78b + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8c1c4f53-e5a1-4131-8adc-053944fb76a9: !Template + answer_choices: null + id: 8c1c4f53-e5a1-4131-8adc-053944fb76a9 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ru-target + reference: '' + 98d4bc66-c822-44cc-a8bc-325c429f48d3: !Template + answer_choices: null + id: 98d4bc66-c822-44cc-a8bc-325c429f48d3 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-source+target + reference: '' + a29bb5d6-fa33-4334-890c-dd565becbfd9: !Template + answer_choices: null + id: a29bb5d6-fa33-4334-890c-dd565becbfd9 + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target + reference: '' + b3025df4-de0b-4cc9-90ca-3acfc31e3af8: !Template + answer_choices: null + id: b3025df4-de0b-4cc9-90ca-3acfc31e3af8 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + d081ca93-c999-41f4-b9d6-d9e5ba83b02e: !Template + answer_choices: null + id: d081ca93-c999-41f4-b9d6-d9e5ba83b02e + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + d8f8de49-13ba-42a9-ab3f-5456c64ff59c: !Template + answer_choices: null + id: d8f8de49-13ba-42a9-ab3f-5456c64ff59c + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-src+target + reference: '' + eb5d5176-d9e6-4172-a951-84ba5d86eee6: !Template + answer_choices: null + id: eb5d5176-d9e6-4172-a951-84ba5d86eee6 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-src+target + reference: '' + ef943a36-5dd4-407e-96bb-04a3298c92d9: !Template + answer_choices: null + id: ef943a36-5dd4-407e-96bb-04a3298c92d9 + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target + reference: '' diff --git a/promptsource/templates/wmt19/zh-en/templates.yaml b/promptsource/templates/wmt19/zh-en/templates.yaml new file mode 100644 index 000000000..d5fedaa22 --- /dev/null +++ b/promptsource/templates/wmt19/zh-en/templates.yaml @@ -0,0 +1,315 @@ +dataset: wmt19 +subset: zh-en +templates: + 10d9af5d-65ad-4744-93f5-5b4ce3ff8355: !Template + answer_choices: null + id: 10d9af5d-65ad-4744-93f5-5b4ce3ff8355 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-target + reference: '' + 469e173c-949f-470b-819b-eec2851b2138: !Template + answer_choices: null + id: 469e173c-949f-470b-819b-eec2851b2138 + jinja: 'Translate English into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-src+starget + reference: '' + 500e5a0b-5bd3-42b6-b5fb-a9b16cce4803: !Template + answer_choices: null + id: 500e5a0b-5bd3-42b6-b5fb-a9b16cce4803 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-source+target + reference: '' + 551f3d20-d549-4c3c-b987-18e47fc05802: !Template + answer_choices: null + id: 551f3d20-d549-4c3c-b987-18e47fc05802 + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-target + reference: '' + 62886e69-7b80-4730-b047-4bace183bad7: !Template + answer_choices: null + id: 62886e69-7b80-4730-b047-4bace183bad7 + jinja: 'Translate into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-zh-target + reference: '' + 6485f138-7c13-4212-ad22-8015c84a2cf4: !Template + answer_choices: null + id: 6485f138-7c13-4212-ad22-8015c84a2cf4 + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-source+target + reference: '' + 6754d78e-d4d1-4812-8fb6-d4e570933622: !Template + answer_choices: null + id: 6754d78e-d4d1-4812-8fb6-d4e570933622 + jinja: ' {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-target + reference: '' + 6dec7703-89bc-4b43-b8a5-d56fa068a97c: !Template + answer_choices: null + id: 6dec7703-89bc-4b43-b8a5-d56fa068a97c + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-source+target + reference: '' + 6ea3018a-4c5f-47c0-9d8c-7879acc7484a: !Template + answer_choices: null + id: 6ea3018a-4c5f-47c0-9d8c-7879acc7484a + jinja: 'English: {{translation["en"]}} translates into Chinese as: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-zh-source+target + reference: '' + 716e1c1a-0bee-4293-ac2e-0e993f810fcf: !Template + answer_choices: null + id: 716e1c1a-0bee-4293-ac2e-0e993f810fcf + jinja: '{{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-target + reference: '' + 739ab695-96ce-450e-b1a4-cac16732210e: !Template + answer_choices: null + id: 739ab695-96ce-450e-b1a4-cac16732210e + jinja: 'How do you say {{translation["zh"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-zh-en-target + reference: '' + 75084cb1-54f6-4fd1-ad11-be0176766173: !Template + answer_choices: null + id: 75084cb1-54f6-4fd1-ad11-be0176766173 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-src+target + reference: '' + 7becef8b-011a-4c22-9159-dd755c749d27: !Template + answer_choices: null + id: 7becef8b-011a-4c22-9159-dd755c749d27 + jinja: 'What is the Chinese translation of: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-target + reference: '' + 8cbf3dde-9545-4863-aabb-3807f9544c0c: !Template + answer_choices: null + id: 8cbf3dde-9545-4863-aabb-3807f9544c0c + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-target + reference: '' + 9616be65-d1e1-462a-a27c-05e420914f52: !Template + answer_choices: null + id: 9616be65-d1e1-462a-a27c-05e420914f52 + jinja: 'How do you say {{translation["en"]}} in Chinese? + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-zh-target + reference: '' + a9feb6c0-a8ac-4f9d-aa1b-971c146925ed: !Template + answer_choices: null + id: a9feb6c0-a8ac-4f9d-aa1b-971c146925ed + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-src+target + reference: '' + ad0b11c7-8fe8-4567-8046-1e817a57988a: !Template + answer_choices: null + id: ad0b11c7-8fe8-4567-8046-1e817a57988a + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-target + reference: '' + b26b8e43-2e1a-4020-8ebc-7f5e8ec959c8: !Template + answer_choices: null + id: b26b8e43-2e1a-4020-8ebc-7f5e8ec959c8 + jinja: 'Translate Chinese into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e3cf11c0-0142-4b35-a518-1df52fc9fa59: !Template + answer_choices: null + id: e3cf11c0-0142-4b35-a518-1df52fc9fa59 + jinja: 'What is the English translation of : {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-target + reference: '' + e622d282-3b67-4a8e-a212-83f128bba643: !Template + answer_choices: null + id: e622d282-3b67-4a8e-a212-83f128bba643 + jinja: 'Translate into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-target + reference: '' + f325cf8b-28ab-4ce6-bafd-f7c2fc05a741: !Template + answer_choices: null + id: f325cf8b-28ab-4ce6-bafd-f7c2fc05a741 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-zh-source+target + reference: '' + f4e0e33d-468d-4bb3-8301-84edc1e579b6: !Template + answer_choices: null + id: f4e0e33d-468d-4bb3-8301-84edc1e579b6 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-source+target + reference: '' From fb9d6efa673ac209fee56f02f8c2f424626ceb54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Thu, 16 Jun 2022 21:26:04 +0200 Subject: [PATCH 13/15] added 4 new glm like prompts for wmt* tasks --- .../templates/wmt14/cs-en/templates.yaml | 262 ++++++++++------ .../templates/wmt14/de-en/templates.yaml | 282 ++++++++++------- .../templates/wmt14/fr-en/templates.yaml | 66 +++- .../templates/wmt14/hi-en/templates.yaml | 280 ++++++++++------- .../templates/wmt14/ru-en/templates.yaml | 274 +++++++++------- .../templates/wmt15/cs-en/templates.yaml | 276 ++++++++++------- .../templates/wmt15/de-en/templates.yaml | 268 +++++++++------- .../templates/wmt15/fi-en/templates.yaml | 268 +++++++++------- .../templates/wmt15/fr-en/templates.yaml | 278 ++++++++++------- .../templates/wmt15/ru-en/templates.yaml | 280 ++++++++++------- .../templates/wmt16/cs-en/templates.yaml | 272 +++++++++------- .../templates/wmt16/de-en/templates.yaml | 270 +++++++++------- .../templates/wmt16/fi-en/templates.yaml | 246 +++++++++------ .../templates/wmt16/ro-en/templates.yaml | 288 ++++++++++------- .../templates/wmt16/ru-en/templates.yaml | 262 ++++++++++------ .../templates/wmt16/tr-en/templates.yaml | 268 +++++++++------- .../templates/wmt17/cs-en/templates.yaml | 272 +++++++++------- .../templates/wmt17/de-en/templates.yaml | 270 +++++++++------- .../templates/wmt17/fi-en/templates.yaml | 288 ++++++++++------- .../templates/wmt17/lv-en/templates.yaml | 278 ++++++++++------- .../templates/wmt17/ru-en/templates.yaml | 268 +++++++++------- .../templates/wmt17/tr-en/templates.yaml | 272 +++++++++------- .../templates/wmt17/zh-en/templates.yaml | 280 ++++++++++------- .../templates/wmt18/cs-en/templates.yaml | 276 ++++++++++------- .../templates/wmt18/de-en/templates.yaml | 290 ++++++++++------- .../templates/wmt18/et-en/templates.yaml | 268 +++++++++------- .../templates/wmt18/fi-en/templates.yaml | 272 +++++++++------- .../templates/wmt18/ru-en/templates.yaml | 278 ++++++++++------- .../templates/wmt18/tr-en/templates.yaml | 270 +++++++++------- .../templates/wmt18/zh-en/templates.yaml | 274 +++++++++------- .../templates/wmt19/de-en/templates.yaml | 292 +++++++++++------- .../templates/wmt19/fi-en/templates.yaml | 270 +++++++++------- .../templates/wmt19/fr-de/templates.yaml | 272 +++++++++------- .../templates/wmt19/gu-en/templates.yaml | 280 ++++++++++------- .../templates/wmt19/kk-en/templates.yaml | 290 ++++++++++------- .../templates/wmt19/lt-en/templates.yaml | 270 +++++++++------- .../templates/wmt19/ru-en/templates.yaml | 282 ++++++++++------- .../templates/wmt19/zh-en/templates.yaml | 284 ++++++++++------- 38 files changed, 6180 insertions(+), 4056 deletions(-) diff --git a/promptsource/templates/wmt14/cs-en/templates.yaml b/promptsource/templates/wmt14/cs-en/templates.yaml index 93a48e5f6..5ee5872e7 100644 --- a/promptsource/templates/wmt14/cs-en/templates.yaml +++ b/promptsource/templates/wmt14/cs-en/templates.yaml @@ -1,24 +1,24 @@ dataset: wmt14 subset: cs-en templates: - 09a0c104-71e9-406d-9ee2-e28ed6e5c9f5: !Template + 1ddc856c-16a7-4a0a-a624-6007fdeab8bd: !Template answer_choices: null - id: 09a0c104-71e9-406d-9ee2-e28ed6e5c9f5 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 1ddc856c-16a7-4a0a-a624-6007fdeab8bd + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-src+target + - translate_as_cs-en-source+target reference: '' - 2bb0c147-5d2c-4644-97a1-24f53e65f203: !Template + 325f58a2-55ca-4486-9f31-6791fc5e6641: !Template answer_choices: null - id: 2bb0c147-5d2c-4644-97a1-24f53e65f203 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 325f58a2-55ca-4486-9f31-6791fc5e6641 + jinja: '{{translation["cs"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -27,41 +27,82 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 39a01670-57d3-482c-b53f-eeebcc09cb1d: !Template + answer_choices: null + id: 39a01670-57d3-482c-b53f-eeebcc09cb1d + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target reference: '' - 3801802f-3fea-4b22-8952-dae4330a8a51: !Template + 4558ada7-1530-4e1b-a8cc-a20f2df5b9da: !Template answer_choices: null - id: 3801802f-3fea-4b22-8952-dae4330a8a51 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: 4558ada7-1530-4e1b-a8cc-a20f2df5b9da + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target + - version-cs-en-target reference: '' - 3b731ca7-79c0-4e28-87ba-ca44dd48ca66: !Template + 5161144a-47c3-4ff5-8f4c-266072f51787: !Template answer_choices: null - id: 3b731ca7-79c0-4e28-87ba-ca44dd48ca66 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: 5161144a-47c3-4ff5-8f4c-266072f51787 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - gpt-3-cs-en-source+target + reference: GPT-3 paper + 560c6739-f946-410a-bd78-f9bc6062a698: !Template + answer_choices: null + id: 560c6739-f946-410a-bd78-f9bc6062a698 + jinja: 'Translate Czech into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 56c643ec-d2d6-4b0f-9e9f-68c7e1fc5b2a: !Template + answer_choices: null + id: 56c643ec-d2d6-4b0f-9e9f-68c7e1fc5b2a + jinja: '{{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-target reference: '' - 42e55c35-5fa3-4b02-89ea-98a418ccf166: !Template + 56cc3bed-3382-4644-93b6-a650315e2c27: !Template answer_choices: null - id: 42e55c35-5fa3-4b02-89ea-98a418ccf166 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 56cc3bed-3382-4644-93b6-a650315e2c27 + jinja: 'What is the Czech translation of: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -70,52 +111,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-src+starget - reference: '' - 4539ae0a-4d2f-4aba-a546-0b0e661799a5: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 5f9590ab-e3bf-446e-a815-9134d15d7a04: !Template answer_choices: null - id: 4539ae0a-4d2f-4aba-a546-0b0e661799a5 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 5f9590ab-e3bf-446e-a815-9134d15d7a04 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target - reference: '' - 4d723760-2c6d-40f9-b894-7517dd9a77bd: !Template + - gpt-3-cs-en-target + reference: GPT-3 paper + 6799e0cd-f0d1-4777-8ce8-da82ed2e8dc7: !Template answer_choices: null - id: 4d723760-2c6d-40f9-b894-7517dd9a77bd - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 6799e0cd-f0d1-4777-8ce8-da82ed2e8dc7 + jinja: 'Translate into English: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - translate-cs-en-target reference: '' - 60e4f2bd-3d89-4d60-bc20-3ed0bd211289: !Template + 79432166-2192-45dc-a3aa-fa75be4456fe: !Template answer_choices: null - id: 60e4f2bd-3d89-4d60-bc20-3ed0bd211289 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: 79432166-2192-45dc-a3aa-fa75be4456fe + jinja: 'Translate into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 6f3e78f6-c332-400c-8b9b-27ef01c98ec6: !Template + - translate-en-cs-target + reference: '' + 85cb5985-5cd9-4c7e-b15d-1eb3fd887c23: !Template answer_choices: null - id: 6f3e78f6-c332-400c-8b9b-27ef01c98ec6 + id: 85cb5985-5cd9-4c7e-b15d-1eb3fd887c23 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Czech translation is: @@ -128,37 +170,38 @@ templates: name: !!python/tuple - a_good_translation-en-cs-source+target reference: '' - 73edc11b-9109-42a0-951f-632cf9d91160: !Template + 8d06286c-0584-4548-b2b6-fe5b4dabae9a: !Template answer_choices: null - id: 73edc11b-9109-42a0-951f-632cf9d91160 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 8d06286c-0584-4548-b2b6-fe5b4dabae9a + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - how_to_say-cs-en-target reference: '' - 774b3b47-c85b-4fff-b47e-3fc6bb6dc223: !Template + 93853dba-752f-4bdc-a572-9dd2470c5ee2: !Template answer_choices: null - id: 774b3b47-c85b-4fff-b47e-3fc6bb6dc223 - jinja: 'Translate into English: {{translation["cs"]}} + id: 93853dba-752f-4bdc-a572-9dd2470c5ee2 + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - how_to_say-en-cs-target reference: '' - 7ea6d7d5-cc1d-4136-857d-00b1b5e37f81: !Template + ae00bea5-318a-496c-a013-72311dac8418: !Template answer_choices: null - id: 7ea6d7d5-cc1d-4136-857d-00b1b5e37f81 - jinja: 'How do you say {{translation["en"]}} in Czech? + id: ae00bea5-318a-496c-a013-72311dac8418 + jinja: ' {{translation["en"]}} translates into Czech as: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -167,11 +210,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_en-cs-target reference: '' - 81fe53c1-08a7-4938-b83b-7ff6d43206d5: !Template + b0fc060e-9c17-4b89-b931-af0027a8fba1: !Template answer_choices: null - id: 81fe53c1-08a7-4938-b83b-7ff6d43206d5 + id: b0fc060e-9c17-4b89-b931-af0027a8fba1 jinja: 'If the original version says: {{translation["en"]}}; then the Czech version should say: @@ -184,52 +227,50 @@ templates: name: !!python/tuple - version-en-cs-target reference: '' - a0b676ea-2118-44f0-b432-c9789f6556bb: !Template + bde02500-7413-46a8-aa2f-438bfa77275d: !Template answer_choices: null - id: a0b676ea-2118-44f0-b432-c9789f6556bb - jinja: '{{translation["cs"]}} translates into English as: + id: bde02500-7413-46a8-aa2f-438bfa77275d + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - translate_as_en-cs-source+target reference: '' - ae09df50-7e04-4dba-bad1-251f74b654c1: !Template + c45fbb7a-dabf-4990-a63e-cf2543050511: !Template answer_choices: null - id: ae09df50-7e04-4dba-bad1-251f74b654c1 - jinja: 'Translate into Czech: {{translation["en"]}} - - ||| {{translation["cs"]}}' + id: c45fbb7a-dabf-4990-a63e-cf2543050511 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target + - a_good_translation-cs-en-source+target reference: '' - b210b8e8-f476-42a7-888c-e0abbbe62eca: !Template + cc19db28-9a53-4ffc-9f98-9e399cba8b33: !Template answer_choices: null - id: b210b8e8-f476-42a7-888c-e0abbbe62eca - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: cc19db28-9a53-4ffc-9f98-9e399cba8b33 + jinja: '{{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target - reference: '' - bcf3a7d3-22ac-4f4c-ac9b-7f228103ff68: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d01581c4-3bfe-4290-9ce6-04f354660f1b: !Template answer_choices: null - id: bcf3a7d3-22ac-4f4c-ac9b-7f228103ff68 + id: d01581c4-3bfe-4290-9ce6-04f354660f1b jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation is: @@ -242,37 +283,38 @@ templates: name: !!python/tuple - a_good_translation-cs-en-target reference: '' - caf37bc6-877c-495d-b7eb-591ab5aa578d: !Template + d13d8c55-6569-48ee-a12e-4a1ed7351652: !Template answer_choices: null - id: caf37bc6-877c-495d-b7eb-591ab5aa578d - jinja: ' {{translation["en"]}} translates into Czech as: + id: d13d8c55-6569-48ee-a12e-4a1ed7351652 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - version-en-cs-source+target reference: '' - cc29a367-412c-4e1c-9b6f-f23703c35858: !Template + e2ff2252-88e6-423d-a992-34f5fcf2f36e: !Template answer_choices: null - id: cc29a367-412c-4e1c-9b6f-f23703c35858 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: e2ff2252-88e6-423d-a992-34f5fcf2f36e + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: '' - e76eb1ad-7fe5-4af6-9dd3-4d85cf158c04: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e87df111-e881-4887-b3b2-ba2fd778b1ce: !Template answer_choices: null - id: e76eb1ad-7fe5-4af6-9dd3-4d85cf158c04 + id: e87df111-e881-4887-b3b2-ba2fd778b1ce jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version should say: @@ -285,24 +327,24 @@ templates: name: !!python/tuple - version-cs-en-source+target reference: '' - f0471cf5-7531-434c-b174-fedac84ae483: !Template + f2b5cc24-d38e-4085-9348-fa2a77b580d9: !Template answer_choices: null - id: f0471cf5-7531-434c-b174-fedac84ae483 - jinja: 'How do you say {{translation["cs"]}} in English? + id: f2b5cc24-d38e-4085-9348-fa2a77b580d9 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target - reference: '' - fbf9be56-438b-42bd-896c-6adbebe11299: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + f6b141d3-36ba-4cbb-a0f5-2d03ecc19a66: !Template answer_choices: null - id: fbf9be56-438b-42bd-896c-6adbebe11299 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: f6b141d3-36ba-4cbb-a0f5-2d03ecc19a66 + jinja: 'Translate English into Czech: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -311,5 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-src+target + - translate-en-cs-source+starget reference: '' + faa45418-ed53-461b-abfb-267790bfea6a: !Template + answer_choices: null + id: faa45418-ed53-461b-abfb-267790bfea6a + jinja: 'English: {{translation["en"]}} = Czech: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml index 7fe063983..f34f9cc99 100644 --- a/promptsource/templates/wmt14/de-en/templates.yaml +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -1,25 +1,37 @@ dataset: wmt14 subset: de-en templates: - 028689e7-d0de-4bc7-902f-6a6d7b523dfe: !Template + 018980f2-de9d-45e5-b2b6-29a68bb4cc12: !Template answer_choices: null - id: 028689e7-d0de-4bc7-902f-6a6d7b523dfe - jinja: 'Translate into English: {{translation["de"]}} + id: 018980f2-de9d-45e5-b2b6-29a68bb4cc12 + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - translate_as_de-en-target reference: '' - 2daaac1b-1431-4b18-98ba-f9e2479a79cf: !Template + 02c6672a-d29b-4604-b60d-74e04317c362: !Template answer_choices: null - id: 2daaac1b-1431-4b18-98ba-f9e2479a79cf - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 02c6672a-d29b-4604-b60d-74e04317c362 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-source+target + reference: '' + 03158c3a-0709-4489-909d-2f2c44b9b7c6: !Template + answer_choices: null + id: 03158c3a-0709-4489-909d-2f2c44b9b7c6 + jinja: 'English: {{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -28,82 +40,84 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target - reference: '' - 3284f6a8-b441-493e-b76c-4e9116d03eaa: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 0bfa68cc-2054-4d31-85df-787b24b0c0c2: !Template answer_choices: null - id: 3284f6a8-b441-493e-b76c-4e9116d03eaa - jinja: '{{translation["de"]}} translates into English as: + id: 0bfa68cc-2054-4d31-85df-787b24b0c0c2 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target - reference: '' - 477d648c-6d23-4757-b21d-1a690e2cb61d: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 12459bda-43d1-4b92-9128-206b49eb4230: !Template answer_choices: null - id: 477d648c-6d23-4757-b21d-1a690e2cb61d - jinja: 'German: {{translation["de"]}} translates into English as: + id: 12459bda-43d1-4b92-9128-206b49eb4230 + jinja: 'Translate German into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target - reference: '' - 47b3f216-954c-4bf9-8347-cc49b9587f7f: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1aed34d3-71c0-4478-acf9-905ac7f68cf9: !Template answer_choices: null - id: 47b3f216-954c-4bf9-8347-cc49b9587f7f - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: 1aed34d3-71c0-4478-acf9-905ac7f68cf9 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - version-de-en-target reference: '' - 4876ebc0-53fe-447c-ac1e-88b8d7fdc1cb: !Template + 22d04cb6-3603-4d49-96f5-227410929d9c: !Template answer_choices: null - id: 4876ebc0-53fe-447c-ac1e-88b8d7fdc1cb - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 22d04cb6-3603-4d49-96f5-227410929d9c + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate_as_en-de-source+target reference: '' - 7035f352-28ed-4025-af47-faf9e60680ba: !Template + 3432fbc9-d1c2-437f-97c9-23c048e39197: !Template answer_choices: null - id: 7035f352-28ed-4025-af47-faf9e60680ba - jinja: 'What is the English translation of : {{translation["de"]}} + id: 3432fbc9-d1c2-437f-97c9-23c048e39197 + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target + - how_to_say-de-en-target reference: '' - 74753485-14e5-4acd-a82b-6eb12a4ac3ea: !Template + 41a8dad3-0d97-4c91-a232-749f28e27697: !Template answer_choices: null - id: 74753485-14e5-4acd-a82b-6eb12a4ac3ea - jinja: 'English: {{translation["en"]}} translates into German as: + id: 41a8dad3-0d97-4c91-a232-749f28e27697 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -112,55 +126,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - a_good_translation-en-de-source+target reference: '' - 787c693b-0759-4b1b-b9ef-3ec9ba9f2c37: !Template + 572f7058-68a2-4c60-aa38-56e4da45b6e6: !Template answer_choices: null - id: 787c693b-0759-4b1b-b9ef-3ec9ba9f2c37 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: 572f7058-68a2-4c60-aa38-56e4da45b6e6 + jinja: ' {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - translate_as_en-de-target reference: '' - 7af42f5e-98e8-4248-a54e-dec21045ba0b: !Template + 5cc7fb46-364b-4494-9557-40e579258f0f: !Template answer_choices: null - id: 7af42f5e-98e8-4248-a54e-dec21045ba0b - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: 5cc7fb46-364b-4494-9557-40e579258f0f + jinja: '{{translation["en"]}} = German: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - 8966ea5e-92ad-4fcb-8aa0-5f2207ba4acb: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 6088a5e1-a753-4d55-a977-6b1fbb3d5ebc: !Template answer_choices: null - id: 8966ea5e-92ad-4fcb-8aa0-5f2207ba4acb - jinja: 'What is the German translation of: {{translation["en"]}} + id: 6088a5e1-a753-4d55-a977-6b1fbb3d5ebc + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target + - how_to_say-en-de-target reference: '' - 8abfaaee-565f-40b9-8a20-6731f35aa0a6: !Template + 68222c7d-2854-4289-9207-33c072b9c382: !Template answer_choices: null - id: 8abfaaee-565f-40b9-8a20-6731f35aa0a6 + id: 68222c7d-2854-4289-9207-33c072b9c382 jinja: 'Translate into German: {{translation["en"]}} ||| {{translation["de"]}}' @@ -172,38 +184,37 @@ templates: name: !!python/tuple - translate-en-de-target reference: '' - 99d4e661-6ec3-4f21-ac52-64a8d3e42374: !Template + 6e98a0d2-595f-4c7a-a253-3ec0325846b5: !Template answer_choices: null - id: 99d4e661-6ec3-4f21-ac52-64a8d3e42374 - jinja: ' {{translation["en"]}} translates into German as: + id: 6e98a0d2-595f-4c7a-a253-3ec0325846b5 + jinja: 'Translate English into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - translate-en-de-source+starget reference: '' - a1598a2a-c3ec-413c-9ae1-727dccec8a88: !Template + 7b8e23c3-a170-4ac9-9fd2-5c8142144f58: !Template answer_choices: null - id: a1598a2a-c3ec-413c-9ae1-727dccec8a88 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - - ||| {{translation["en"]}}' + id: 7b8e23c3-a170-4ac9-9fd2-5c8142144f58 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target + - a_good_translation-en-de-target reference: '' - a85989e5-787e-4953-9f29-1e1b453a7413: !Template + 7e21750f-6aa2-4893-9b1a-5f6b30dc9b4d: !Template answer_choices: null - id: a85989e5-787e-4953-9f29-1e1b453a7413 - jinja: 'Translate English into German: {{translation["en"]}} + id: 7e21750f-6aa2-4893-9b1a-5f6b30dc9b4d + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -212,12 +223,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget - reference: '' - aaf593d0-4bc8-4d35-9f14-c2545ae77d6b: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 87400fa8-12d7-4ca2-b843-e45c9c9bdefc: !Template answer_choices: null - id: aaf593d0-4bc8-4d35-9f14-c2545ae77d6b - jinja: 'Translate German into English: {{translation["de"]}} + id: 87400fa8-12d7-4ca2-b843-e45c9c9bdefc + jinja: '{{translation["de"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9ce7e8f7-c563-44b2-85bf-e3ae36e2ced0: !Template + answer_choices: null + id: 9ce7e8f7-c563-44b2-85bf-e3ae36e2ced0 + jinja: 'Translate into English: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -226,11 +251,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - c11e1bd0-a07d-4cd5-9873-621dc5764fda: !Template + - translate-de-en-target + reference: '' + a77aebb0-b708-466a-8866-138fc1813b17: !Template answer_choices: null - id: c11e1bd0-a07d-4cd5-9873-621dc5764fda + id: a77aebb0-b708-466a-8866-138fc1813b17 jinja: 'Given the following passage: {{translation["de"]}} , a good English translation is: @@ -243,66 +268,68 @@ templates: name: !!python/tuple - a_good_translation-de-en-target reference: '' - cd13a8e4-26c8-4f51-95ea-04025752ebb2: !Template + b306075f-da0c-4d28-98eb-20d4472987dc: !Template answer_choices: null - id: cd13a8e4-26c8-4f51-95ea-04025752ebb2 - jinja: 'How do you say {{translation["en"]}} in German? + id: b306075f-da0c-4d28-98eb-20d4472987dc + jinja: 'What is the English translation of : {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - dc9f3ef3-eab9-4fb0-b825-35ce78dcb835: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + c468702f-5ba2-49e9-a1c7-bddc3367a4be: !Template answer_choices: null - id: dc9f3ef3-eab9-4fb0-b825-35ce78dcb835 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: c468702f-5ba2-49e9-a1c7-bddc3367a4be + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - version-en-de-target reference: '' - eb2dff5a-8097-48e6-a24a-fc5f0281aa20: !Template + cec39bc6-0742-4a8c-9e27-dcecfde865cb: !Template answer_choices: null - id: eb2dff5a-8097-48e6-a24a-fc5f0281aa20 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: cec39bc6-0742-4a8c-9e27-dcecfde865cb + jinja: 'German: {{translation["de"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target - reference: '' - f927c8f6-2cf4-47ed-8641-124912eea736: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d3db7a9c-7021-49eb-a7ec-df93b3c4c731: !Template answer_choices: null - id: f927c8f6-2cf4-47ed-8641-124912eea736 - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: d3db7a9c-7021-49eb-a7ec-df93b3c4c731 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target + - version-en-de-source+target reference: '' - f9bb17d8-3e15-4865-9f2e-bd23077ea77c: !Template + dfb9e96b-436f-4c19-a746-ae5f3f1145c8: !Template answer_choices: null - id: f9bb17d8-3e15-4865-9f2e-bd23077ea77c - jinja: 'How do you say {{translation["de"]}} in English? + id: dfb9e96b-436f-4c19-a746-ae5f3f1145c8 + jinja: 'German: {{translation["de"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +338,34 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - translate_as_de-en-source+target reference: '' + e4601e49-0d41-4f89-af5e-e5866a618f75: !Template + answer_choices: null + id: e4601e49-0d41-4f89-af5e-e5866a618f75 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target + reference: '' + fb86ef86-ea62-48b2-bb6f-d63d19f86510: !Template + answer_choices: null + id: fb86ef86-ea62-48b2-bb6f-d63d19f86510 + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index 48fb73e19..f2d6a2e5a 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -53,8 +53,8 @@ templates: metrics: - BLEU original_task: true - name: gpt-3-fr-en-src+target - reference: '' + name: gpt-3-fr-en-source+target + reference: 'GPT-3 paper' 280bc983-9f20-4e80-ab02-5f333df90a42: !Template answer_choices: null id: 280bc983-9f20-4e80-ab02-5f333df90a42 @@ -67,7 +67,20 @@ templates: - BLEU original_task: true name: gpt-3-fr-en-target - reference: '' + reference: 'GPT-3 paper' + 2fc841fb-b872-4cc6-9a88-735d6bb7e2e3: !Template + answer_choices: null + id: 2fc841fb-b872-4cc6-9a88-735d6bb7e2e3 + jinja: '{{translation["en"]}} = French: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: xglm-en-fr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 41e7647e-2b9f-4d86-8987-5abe70000362: !Template answer_choices: null id: 41e7647e-2b9f-4d86-8987-5abe70000362 @@ -117,7 +130,7 @@ templates: - BLEU original_task: true name: gpt-3-en-fr-target - reference: '' + reference: 'GPT-3 paper' 51d41a04-1d90-4bae-b6e5-be7598cdbfb0: !Template answer_choices: null id: 51d41a04-1d90-4bae-b6e5-be7598cdbfb0 @@ -155,7 +168,7 @@ templates: metrics: - BLEU original_task: true - name: translate-en-fr-src+starget + name: translate-en-fr-source+starget reference: '' 762c0878-c8fc-43ec-839f-d5d8435a94f6: !Template answer_choices: null @@ -171,6 +184,32 @@ templates: original_task: true name: a_good_translation-fr-en-target reference: '' + 8713594d-626b-4c5c-a63f-553ddc5444de: !Template + answer_choices: null + id: 8713594d-626b-4c5c-a63f-553ddc5444de + jinja: 'English: {{translation["en"]}} = French: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: xglm-en-fr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 957b8554-a00a-4652-b080-e9ee3ccae381: !Template + answer_choices: null + id: 957b8554-a00a-4652-b080-e9ee3ccae381 + jinja: '{{translation["fr"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: xglm-fr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 9fe6b44b-2dc6-4557-8201-14d6ea7668ff: !Template answer_choices: null id: 9fe6b44b-2dc6-4557-8201-14d6ea7668ff @@ -225,6 +264,19 @@ templates: original_task: true name: translate_as_en-fr-source+target reference: '' + b0e110eb-4fde-45fd-a561-e8e24ad83916: !Template + answer_choices: null + id: b0e110eb-4fde-45fd-a561-e8e24ad83916 + jinja: 'French: {{translation["fr"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: xglm-fr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 bc95c647-5979-4522-9e6e-f5eb93b69a00: !Template answer_choices: null id: bc95c647-5979-4522-9e6e-f5eb93b69a00 @@ -289,5 +341,5 @@ templates: metrics: - BLEU original_task: true - name: gpt-3-en-fr-src+target - reference: '' + name: gpt-3-en-fr-source+target + reference: 'GPT 3 paper' diff --git a/promptsource/templates/wmt14/hi-en/templates.yaml b/promptsource/templates/wmt14/hi-en/templates.yaml index c33cf783a..c7475686f 100644 --- a/promptsource/templates/wmt14/hi-en/templates.yaml +++ b/promptsource/templates/wmt14/hi-en/templates.yaml @@ -1,178 +1,178 @@ dataset: wmt14 subset: hi-en templates: - 25a240a7-6de9-4489-a4fd-baf6660936c7: !Template + 0597ecbe-44c0-40ad-b22c-e52ac7d28f03: !Template answer_choices: null - id: 25a240a7-6de9-4489-a4fd-baf6660936c7 - jinja: 'Translate Hindi into English: {{translation["hi"]}} + id: 0597ecbe-44c0-40ad-b22c-e52ac7d28f03 + jinja: '{{translation["en"]}} = Hindi: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-hi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 31ef5fd9-a307-4c13-bd76-ac27fb6dc93f: !Template + - xglm-en-hi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 32166763-eb01-41a1-8793-51125460f5cc: !Template answer_choices: null - id: 31ef5fd9-a307-4c13-bd76-ac27fb6dc93f - jinja: ' {{translation["en"]}} translates into Hindi as: + id: 32166763-eb01-41a1-8793-51125460f5cc + jinja: 'How do you say {{translation["hi"]}} in English? - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-hi-target + - how_to_say-hi-en-target reference: '' - 3ab54e14-30fd-452e-84e2-76b0503a37a1: !Template + 3be84903-3560-4481-b35c-486daefecd67: !Template answer_choices: null - id: 3ab54e14-30fd-452e-84e2-76b0503a37a1 - jinja: 'Translate into Hindi: {{translation["en"]}} - - ||| {{translation["hi"]}}' + id: 3be84903-3560-4481-b35c-486daefecd67 + jinja: 'Given the following passage: {{translation["en"]}} , a good Hindi translation + is: ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-hi-target + - a_good_translation-en-hi-target reference: '' - 3c3c0a1f-e0f0-4580-9a7c-c6911510dce0: !Template + 3d857ce6-7f1f-440f-802c-72e52fbf3bdd: !Template answer_choices: null - id: 3c3c0a1f-e0f0-4580-9a7c-c6911510dce0 - jinja: 'Translate English into Hindi: {{translation["en"]}} + id: 3d857ce6-7f1f-440f-802c-72e52fbf3bdd + jinja: 'English: {{translation["en"]}} translates into Hindi as: - ||| {{translation["hi"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-hi-src+starget + - translate_as_en-hi-source+target reference: '' - 4278f720-6fc5-400e-8c96-9999156506d0: !Template + 4094f562-64c5-48fe-b413-6d8fde1084ee: !Template answer_choices: null - id: 4278f720-6fc5-400e-8c96-9999156506d0 - jinja: 'How do you say {{translation["en"]}} in Hindi? + id: 4094f562-64c5-48fe-b413-6d8fde1084ee + jinja: 'If the original version says: {{translation["hi"]}}; then the English + version should say: - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-hi-target + - version-hi-en-target reference: '' - 4d7d835f-188e-40cb-b6df-24f8332828a5: !Template + 436f9f1c-916a-40ae-9b75-3db5d3a711f3: !Template answer_choices: null - id: 4d7d835f-188e-40cb-b6df-24f8332828a5 - jinja: 'Given the following passage: {{translation["en"]}} , a good Hindi translation - is: ||| {{translation["hi"]}}' + id: 436f9f1c-916a-40ae-9b75-3db5d3a711f3 + jinja: '{{translation["hi"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-hi-target - reference: '' - 54360371-c0c7-453a-b5bb-f4976ba885da: !Template + - xglm-hi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4c21064e-83e2-41b3-a0db-45373e12fa26: !Template answer_choices: null - id: 54360371-c0c7-453a-b5bb-f4976ba885da - jinja: 'Translate into English: {{translation["hi"]}} + id: 4c21064e-83e2-41b3-a0db-45373e12fa26 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Hindi translation is: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-hi-en-target + - a_good_translation-en-hi-source+target reference: '' - 5e937fd7-6acb-4a5f-a72a-6e64f986f521: !Template + 58e9f511-1be7-4255-abed-7b97d6197036: !Template answer_choices: null - id: 5e937fd7-6acb-4a5f-a72a-6e64f986f521 - jinja: 'What is the Hindi translation of: {{translation["en"]}} + id: 58e9f511-1be7-4255-abed-7b97d6197036 + jinja: 'Hindi: {{translation["hi"]}} translates into English as: - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-hi-target + - translate_as_hi-en-source+target reference: '' - 5f066596-f2cf-4077-bf44-e591642c8d1f: !Template + 7cf403e3-5b6c-44fc-9046-57bfbc92e05e: !Template answer_choices: null - id: 5f066596-f2cf-4077-bf44-e591642c8d1f - jinja: 'If the Hindi version says: {{translation["hi"]}}; then the English version - should say: + id: 7cf403e3-5b6c-44fc-9046-57bfbc92e05e + jinja: 'Translate Hindi into English: {{translation["hi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-hi-en-source+target - reference: '' - 6ba3186d-4494-495a-aaa1-481fb833d70a: !Template + - translate-hi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 819805c2-ca6b-4a2f-8c67-db22b92210f4: !Template answer_choices: null - id: 6ba3186d-4494-495a-aaa1-481fb833d70a - jinja: 'What is the Hindi translation of the English sentence: {{translation["en"]}} + id: 819805c2-ca6b-4a2f-8c67-db22b92210f4 + jinja: 'How do you say {{translation["en"]}} in Hindi? - ||| {{translation["hi"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-hi-src+target + - how_to_say-en-hi-target reference: '' - 708c4c05-b17b-4cd7-b27d-33de238c00f0: !Template + 914321b1-e420-426e-b491-b1d9869d50d6: !Template answer_choices: null - id: 708c4c05-b17b-4cd7-b27d-33de238c00f0 - jinja: 'If the original version says: {{translation["en"]}}; then the Hindi version - should say: + id: 914321b1-e420-426e-b491-b1d9869d50d6 + jinja: 'What is the English translation of the Hindi sentence: {{translation["hi"]}} - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-hi-target - reference: '' - 83cd298c-4b6a-4362-909d-81a10e83ed0a: !Template + - gpt-3-hi-en-source+target + reference: GPT-3 paper + 94fbd505-da03-4e7d-a16f-2a48ba523e2a: !Template answer_choices: null - id: 83cd298c-4b6a-4362-909d-81a10e83ed0a - jinja: 'What is the English translation of the Hindi sentence: {{translation["hi"]}} + id: 94fbd505-da03-4e7d-a16f-2a48ba523e2a + jinja: 'Translate English into Hindi: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-hi-en-src+target + - translate-en-hi-source+starget reference: '' - 83fa1f01-86cf-432a-b9c8-bcb876e2613c: !Template + 9eae8a37-27be-42de-82ed-b32998f719ce: !Template answer_choices: null - id: 83fa1f01-86cf-432a-b9c8-bcb876e2613c + id: 9eae8a37-27be-42de-82ed-b32998f719ce jinja: '{{translation["hi"]}} translates into English as: ||| {{translation["en"]}}' @@ -184,11 +184,11 @@ templates: name: !!python/tuple - translate_as_hi-en-target reference: '' - 8808a912-237e-43ec-94f2-550c0a234f0f: !Template + a20352ec-8ef1-42ed-8827-001374fc1ee5: !Template answer_choices: null - id: 8808a912-237e-43ec-94f2-550c0a234f0f - jinja: 'If the original version says: {{translation["hi"]}}; then the English - version should say: + id: a20352ec-8ef1-42ed-8827-001374fc1ee5 + jinja: 'If the Hindi version says: {{translation["hi"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -197,42 +197,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-hi-en-target + - version-hi-en-source+target reference: '' - 9acf9775-2bc2-4ecb-bace-7a3dbc6ebce7: !Template + af072fb1-3955-4b6f-8489-ed2d0c9c61cd: !Template answer_choices: null - id: 9acf9775-2bc2-4ecb-bace-7a3dbc6ebce7 - jinja: 'If the English version says: {{translation["en"]}}; then the Hindi version - should say: + id: af072fb1-3955-4b6f-8489-ed2d0c9c61cd + jinja: 'Translate into Hindi: {{translation["en"]}} - ||| {{translation["hi"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-hi-source+target + - translate-en-hi-target reference: '' - 9d6b3e39-21d9-4c44-b1e4-7f808b37de7e: !Template + b134b1da-2286-4d38-bc3d-1f66c92de1cb: !Template answer_choices: null - id: 9d6b3e39-21d9-4c44-b1e4-7f808b37de7e - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Hindi translation is: + id: b134b1da-2286-4d38-bc3d-1f66c92de1cb + jinja: 'What is the English translation of : {{translation["hi"]}} - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-hi-source+target - reference: '' - adca374e-8d97-46c9-a6d7-544e100cbf38: !Template + - gpt-3-hi-en-target + reference: GPT-3 paper + b91acbc0-81fd-498a-aef5-e61ae66d659b: !Template answer_choices: null - id: adca374e-8d97-46c9-a6d7-544e100cbf38 - jinja: 'English: {{translation["en"]}} translates into Hindi as: + id: b91acbc0-81fd-498a-aef5-e61ae66d659b + jinja: ' {{translation["en"]}} translates into Hindi as: ||| {{translation["hi"]}}' metadata: !TemplateMetadata @@ -241,12 +239,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-hi-source+target + - translate_as_en-hi-target reference: '' - aee2cfda-d9c6-423c-a6a3-a189731e5acf: !Template + cc2e0824-a3a8-4ed1-bde2-9b3b893eead3: !Template answer_choices: null - id: aee2cfda-d9c6-423c-a6a3-a189731e5acf - jinja: 'How do you say {{translation["hi"]}} in English? + id: cc2e0824-a3a8-4ed1-bde2-9b3b893eead3 + jinja: 'Hindi: {{translation["hi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -255,25 +253,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-hi-en-target + - xglm-hi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + cda44776-363a-4a2d-aceb-07416ff531f1: !Template + answer_choices: null + id: cda44776-363a-4a2d-aceb-07416ff531f1 + jinja: 'Translate into English: {{translation["hi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-hi-en-target reference: '' - e64fb9f5-2c2d-44de-9d78-ffe8b73e2fc0: !Template + e2b4e89e-d1f4-4655-8147-7fa3839a6f0b: !Template answer_choices: null - id: e64fb9f5-2c2d-44de-9d78-ffe8b73e2fc0 - jinja: 'Hindi: {{translation["hi"]}} translates into English as: + id: e2b4e89e-d1f4-4655-8147-7fa3839a6f0b + jinja: 'If the English version says: {{translation["en"]}}; then the Hindi version + should say: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_hi-en-source+target + - version-en-hi-source+target reference: '' - e7c5870c-5fab-4bdf-b2b2-18cabe12fb15: !Template + e5025d4c-85e9-49c9-9c44-f3b3da3b9710: !Template answer_choices: null - id: e7c5870c-5fab-4bdf-b2b2-18cabe12fb15 + id: e5025d4c-85e9-49c9-9c44-f3b3da3b9710 jinja: 'Given the following passage: {{translation["hi"]}} , a good English translation is: @@ -286,30 +299,73 @@ templates: name: !!python/tuple - a_good_translation-hi-en-target reference: '' - fca8e2a5-d633-4435-8fa8-a3727511619e: !Template + f146c715-e2b8-4952-aa26-7df96c37e73b: !Template answer_choices: null - id: fca8e2a5-d633-4435-8fa8-a3727511619e - jinja: 'Given the following source text in Hindi: {{translation["hi"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: f146c715-e2b8-4952-aa26-7df96c37e73b + jinja: 'What is the Hindi translation of: {{translation["en"]}} + + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-hi-en-source+target + - gpt-3-en-hi-target + reference: GPT-3 paper + f72a0b57-1dce-4e87-a43d-0ee0deadcf11: !Template + answer_choices: null + id: f72a0b57-1dce-4e87-a43d-0ee0deadcf11 + jinja: 'If the original version says: {{translation["en"]}}; then the Hindi version + should say: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-hi-target reference: '' - fed488ee-6048-4e3c-869f-0c66d678c7ab: !Template + f96c3c81-02fd-4435-b09e-7a5dc161e612: !Template answer_choices: null - id: fed488ee-6048-4e3c-869f-0c66d678c7ab - jinja: 'What is the English translation of : {{translation["hi"]}} + id: f96c3c81-02fd-4435-b09e-7a5dc161e612 + jinja: 'English: {{translation["en"]}} = Hindi: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-hi-en-target + - xglm-en-hi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + fef51115-c777-4eb6-8eeb-8b71f227b201: !Template + answer_choices: null + id: fef51115-c777-4eb6-8eeb-8b71f227b201 + jinja: 'What is the Hindi translation of the English sentence: {{translation["en"]}} + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-hi-source+target + reference: GPT 3 paper + ff581230-e0a6-45a5-9c87-f5b89872e2da: !Template + answer_choices: null + id: ff581230-e0a6-45a5-9c87-f5b89872e2da + jinja: 'Given the following source text in Hindi: {{translation["hi"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-hi-en-source+target reference: '' diff --git a/promptsource/templates/wmt14/ru-en/templates.yaml b/promptsource/templates/wmt14/ru-en/templates.yaml index 82bc4af3e..a67763e62 100644 --- a/promptsource/templates/wmt14/ru-en/templates.yaml +++ b/promptsource/templates/wmt14/ru-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt14 subset: ru-en templates: - 0026c4d6-5b4d-440f-ba9a-df7894b4f92c: !Template + 073e3350-9315-4a9d-8330-091354036446: !Template answer_choices: null - id: 0026c4d6-5b4d-440f-ba9a-df7894b4f92c + id: 073e3350-9315-4a9d-8330-091354036446 jinja: 'If the original version says: {{translation["en"]}}; then the Russian version should say: @@ -16,82 +16,82 @@ templates: name: !!python/tuple - version-en-ru-target reference: '' - 2e565e0b-0beb-4a22-a603-7f4fa96123d2: !Template + 17d195b0-cc2f-4369-a578-6cdeb435289a: !Template answer_choices: null - id: 2e565e0b-0beb-4a22-a603-7f4fa96123d2 - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 17d195b0-cc2f-4369-a578-6cdeb435289a + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target + - version-ru-en-source+target reference: '' - 30e887af-c0e6-4063-9e60-4bc226cbfd3e: !Template + 19f6f139-8914-4d0b-be91-7ce530bca4ca: !Template answer_choices: null - id: 30e887af-c0e6-4063-9e60-4bc226cbfd3e - jinja: 'Translate into English: {{translation["ru"]}} + id: 19f6f139-8914-4d0b-be91-7ce530bca4ca + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - translate_as_ru-en-source+target reference: '' - 4738c193-8c27-42ca-9217-20d16bfe892c: !Template + 1a198507-ff2a-458a-aa01-c758468cc87d: !Template answer_choices: null - id: 4738c193-8c27-42ca-9217-20d16bfe892c - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 1a198507-ff2a-458a-aa01-c758468cc87d + jinja: 'Translate Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: '' - 53558679-249a-4205-896f-0b139fc9bd55: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1ec62a81-d73d-4987-a874-477c39d0924e: !Template answer_choices: null - id: 53558679-249a-4205-896f-0b139fc9bd55 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 1ec62a81-d73d-4987-a874-477c39d0924e + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - version-ru-en-target reference: '' - 539ef9e1-4d07-40a6-a2f5-80dad964e2b7: !Template + 2156cb8f-f81e-4a26-a1ec-6b8f94c76dcd: !Template answer_choices: null - id: 539ef9e1-4d07-40a6-a2f5-80dad964e2b7 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 2156cb8f-f81e-4a26-a1ec-6b8f94c76dcd + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - translate_as_en-ru-target reference: '' - 57ff898b-9254-4d5d-a002-affd6f41fe9f: !Template + 2299e8ae-d964-4059-8d7b-61dfc5e02d9a: !Template answer_choices: null - id: 57ff898b-9254-4d5d-a002-affd6f41fe9f - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 2299e8ae-d964-4059-8d7b-61dfc5e02d9a + jinja: 'How do you say {{translation["en"]}} in Russian? ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -100,54 +100,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - how_to_say-en-ru-target reference: '' - 606cfce9-21eb-4f04-b543-563750d4c6d3: !Template + 29874269-2e7d-4dee-82f6-2b64941a808b: !Template answer_choices: null - id: 606cfce9-21eb-4f04-b543-563750d4c6d3 - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 29874269-2e7d-4dee-82f6-2b64941a808b + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: '' - 643776a7-0520-4d5c-864d-f5c6a8342b38: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3162c64d-79f2-4ca8-9b0e-f1b20cd7b66b: !Template answer_choices: null - id: 643776a7-0520-4d5c-864d-f5c6a8342b38 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 3162c64d-79f2-4ca8-9b0e-f1b20cd7b66b + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 65a3b705-0dcc-4283-b585-763ab682db4a: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 435ca8a6-015d-4e87-920e-c7063de43871: !Template answer_choices: null - id: 65a3b705-0dcc-4283-b585-763ab682db4a - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 435ca8a6-015d-4e87-920e-c7063de43871 + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' - 7cd72da5-9047-4712-b2f2-75d04a0b9457: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 52e615f9-bb28-47c0-93d8-68474bf50e26: !Template answer_choices: null - id: 7cd72da5-9047-4712-b2f2-75d04a0b9457 + id: 52e615f9-bb28-47c0-93d8-68474bf50e26 jinja: 'How do you say {{translation["ru"]}} in English? ||| {{translation["en"]}}' @@ -159,38 +158,65 @@ templates: name: !!python/tuple - how_to_say-ru-en-target reference: '' - 82a01d5b-4bff-41a0-9cd4-9d76cabce590: !Template + 576fd9ec-e97e-4077-b533-433dcc9a12d6: !Template answer_choices: null - id: 82a01d5b-4bff-41a0-9cd4-9d76cabce590 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 576fd9ec-e97e-4077-b533-433dcc9a12d6 + jinja: 'Translate into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target + - translate-en-ru-target reference: '' - 8a288d1f-757e-4aa7-a7ac-66bb01c29f65: !Template + 78fca8ed-a720-482e-9a8b-8b45308deda7: !Template answer_choices: null - id: 8a288d1f-757e-4aa7-a7ac-66bb01c29f65 - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: 78fca8ed-a720-482e-9a8b-8b45308deda7 + jinja: 'English: {{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 876e475c-7199-4ad9-8a14-dd8a367fadd9: !Template + answer_choices: null + id: 876e475c-7199-4ad9-8a14-dd8a367fadd9 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 97c1cf42-b71c-4c97-8700-597d474defdb: !Template + answer_choices: null + id: 97c1cf42-b71c-4c97-8700-597d474defdb + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target reference: '' - 8c8d404c-1c7b-4612-8764-c1c905982c17: !Template + 9a39275f-9615-4d33-93b0-99e3d0d49c06: !Template answer_choices: null - id: 8c8d404c-1c7b-4612-8764-c1c905982c17 + id: 9a39275f-9615-4d33-93b0-99e3d0d49c06 jinja: 'Translate English into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' @@ -200,26 +226,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget + - translate-en-ru-source+starget reference: '' - 9564f76a-d551-4d14-89e1-0a532327a0cb: !Template + 9eca671d-7be7-492d-9652-4175f0c98b6d: !Template answer_choices: null - id: 9564f76a-d551-4d14-89e1-0a532327a0cb - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 9eca671d-7be7-492d-9652-4175f0c98b6d + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - version-en-ru-source+target reference: '' - a3e7bc85-f1c3-41bf-b4b3-d3dbaba820e9: !Template + 9f0d7c20-f019-49af-8586-f4db2ab8b502: !Template answer_choices: null - id: a3e7bc85-f1c3-41bf-b4b3-d3dbaba820e9 - jinja: ' {{translation["en"]}} translates into Russian as: + id: 9f0d7c20-f019-49af-8586-f4db2ab8b502 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -228,12 +255,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate_as_en-ru-source+target reference: '' - b242c63d-1530-47f5-8df1-0d403fccb44b: !Template + a3c545e6-2570-42e1-9928-ea7044fafd11: !Template answer_choices: null - id: b242c63d-1530-47f5-8df1-0d403fccb44b - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: a3c545e6-2570-42e1-9928-ea7044fafd11 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-source+target + reference: GPT-3 paper + b44adeb1-00f6-43eb-8344-f7ecb285a315: !Template + answer_choices: null + id: b44adeb1-00f6-43eb-8344-f7ecb285a315 + jinja: '{{translation["ru"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -242,12 +283,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target - reference: '' - b82523d6-80c0-42fd-a39a-b8001b67b29a: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + bab892cc-165b-45c5-b506-b71551f9d5c1: !Template answer_choices: null - id: b82523d6-80c0-42fd-a39a-b8001b67b29a - jinja: '{{translation["ru"]}} translates into English as: + id: bab892cc-165b-45c5-b506-b71551f9d5c1 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -256,60 +298,74 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - a_good_translation-ru-en-target reference: '' - bfbb1dfa-5a7d-4e9c-bebb-49dc6e132eb0: !Template + c16ff5df-b71d-49ee-802c-2420fa01458c: !Template answer_choices: null - id: bfbb1dfa-5a7d-4e9c-bebb-49dc6e132eb0 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: c16ff5df-b71d-49ee-802c-2420fa01458c + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - a_good_translation-en-ru-source+target reference: '' - d794ed7f-d098-4802-8328-09244186cec8: !Template + e5e1dbc2-1677-4155-b140-001f3ad9b952: !Template answer_choices: null - id: d794ed7f-d098-4802-8328-09244186cec8 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: e5e1dbc2-1677-4155-b140-001f3ad9b952 + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - gpt-3-ru-en-target + reference: GPT-3 paper + e6ba8d1c-4d94-481c-82ff-d3f72e34e2e9: !Template + answer_choices: null + id: e6ba8d1c-4d94-481c-82ff-d3f72e34e2e9 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target reference: '' - f568cbb6-e2a6-4cb2-b51d-cdb36e50df18: !Template + ed200eb0-0785-4c81-b4ea-16388cb562f9: !Template answer_choices: null - id: f568cbb6-e2a6-4cb2-b51d-cdb36e50df18 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: ed200eb0-0785-4c81-b4ea-16388cb562f9 + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - translate_as_ru-en-target reference: '' - fe8533e6-df7f-46c4-b67e-e0657cd199d2: !Template + fa56517e-e8b7-476d-a05d-e24125ee2128: !Template answer_choices: null - id: fe8533e6-df7f-46c4-b67e-e0657cd199d2 - jinja: 'Translate into Russian: {{translation["en"]}} - - ||| {{translation["ru"]}}' + id: fa56517e-e8b7-476d-a05d-e24125ee2128 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - a_good_translation-ru-en-source+target reference: '' diff --git a/promptsource/templates/wmt15/cs-en/templates.yaml b/promptsource/templates/wmt15/cs-en/templates.yaml index 7de8313fd..ab8b2671e 100644 --- a/promptsource/templates/wmt15/cs-en/templates.yaml +++ b/promptsource/templates/wmt15/cs-en/templates.yaml @@ -1,24 +1,24 @@ dataset: wmt15 subset: cs-en templates: - 06f0d97a-6082-43f4-b581-d59d31e5bce9: !Template + 263d7ce6-7ef1-46e0-bf0e-c63df3dab2e4: !Template answer_choices: null - id: 06f0d97a-6082-43f4-b581-d59d31e5bce9 - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 263d7ce6-7ef1-46e0-bf0e-c63df3dab2e4 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target - reference: '' - 0fbce2e7-b196-4995-a774-de4464474b54: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + 2bc97934-db49-47c6-8391-7fcaa8ed502e: !Template answer_choices: null - id: 0fbce2e7-b196-4995-a774-de4464474b54 - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 2bc97934-db49-47c6-8391-7fcaa8ed502e + jinja: ' {{translation["en"]}} translates into Czech as: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -27,11 +27,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_en-cs-target reference: '' - 1dd6f3bc-a34d-4c02-a62e-3a2c7af9f160: !Template + 3d9952e5-1b15-4fba-896a-a665922628f9: !Template answer_choices: null - id: 1dd6f3bc-a34d-4c02-a62e-3a2c7af9f160 + id: 3d9952e5-1b15-4fba-896a-a665922628f9 jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -42,153 +42,150 @@ templates: name: !!python/tuple - a_good_translation-cs-en-source+target reference: '' - 2b02f0d7-6940-4503-a0b8-7fdb6473f688: !Template + 5abeff8a-09ee-4908-9a87-d715e991178e: !Template answer_choices: null - id: 2b02f0d7-6940-4503-a0b8-7fdb6473f688 - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: 5abeff8a-09ee-4908-9a87-d715e991178e + jinja: 'How do you say {{translation["cs"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - how_to_say-cs-en-target reference: '' - 31bf20ab-ac14-4da5-ae90-46028b960beb: !Template + 7dcdff65-9c00-462e-bf46-467355327d91: !Template answer_choices: null - id: 31bf20ab-ac14-4da5-ae90-46028b960beb - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Czech translation is: - - ||| {{translation["cs"]}}' + id: 7dcdff65-9c00-462e-bf46-467355327d91 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-source+target + - a_good_translation-en-cs-target reference: '' - 3237a771-40b9-476c-811c-36c59194524f: !Template + 809ba8b9-a275-406f-a9cb-fc2a025392c3: !Template answer_choices: null - id: 3237a771-40b9-476c-811c-36c59194524f - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: 809ba8b9-a275-406f-a9cb-fc2a025392c3 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - version-cs-en-source+target reference: '' - 3545b365-136e-4a8b-ab6f-1be700ee4d5e: !Template + 867093b0-5f01-48dc-a2e5-e03bf84e92b1: !Template answer_choices: null - id: 3545b365-136e-4a8b-ab6f-1be700ee4d5e - jinja: '{{translation["cs"]}} translates into English as: + id: 867093b0-5f01-48dc-a2e5-e03bf84e92b1 + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target - reference: '' - 5f8f7d5c-7ca9-40f9-b979-382eb911ca9a: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 8bb2be08-2872-4ac9-9f8f-93f54e65a98a: !Template answer_choices: null - id: 5f8f7d5c-7ca9-40f9-b979-382eb911ca9a - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 8bb2be08-2872-4ac9-9f8f-93f54e65a98a + jinja: 'English: {{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target - reference: '' - 668cd91b-377c-4c36-82d3-ee7eeab17948: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 915e8de4-588d-475a-8930-ed5e65f2109d: !Template answer_choices: null - id: 668cd91b-377c-4c36-82d3-ee7eeab17948 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 915e8de4-588d-475a-8930-ed5e65f2109d + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - version-en-cs-source+target reference: '' - 69eb76fd-ec6c-42ec-9459-6946f86fef81: !Template + 957f1d66-dd9b-4b4e-b8ce-ddb268db329b: !Template answer_choices: null - id: 69eb76fd-ec6c-42ec-9459-6946f86fef81 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 957f1d66-dd9b-4b4e-b8ce-ddb268db329b + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-src+target + - translate_as_en-cs-source+target reference: '' - 6c5e1d66-4521-436e-b609-34b1b3c714f8: !Template + a0649918-ebc1-437f-a90f-e7e4c5ea6e4d: !Template answer_choices: null - id: 6c5e1d66-4521-436e-b609-34b1b3c714f8 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: a0649918-ebc1-437f-a90f-e7e4c5ea6e4d + jinja: 'Translate English into Czech: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - translate-en-cs-source+starget reference: '' - 7d0ad92c-50e7-4be7-9557-aa5f18fd2b0f: !Template + a1207337-11af-4f19-a57f-cad2982cf0fd: !Template answer_choices: null - id: 7d0ad92c-50e7-4be7-9557-aa5f18fd2b0f - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: a1207337-11af-4f19-a57f-cad2982cf0fd + jinja: 'How do you say {{translation["en"]}} in Czech? + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - how_to_say-en-cs-target reference: '' - 85ea7e61-2ee1-4993-8f9f-a870ef0bd140: !Template + a1daba28-90ab-40f6-a222-d7adf8b98034: !Template answer_choices: null - id: 85ea7e61-2ee1-4993-8f9f-a870ef0bd140 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: a1daba28-90ab-40f6-a222-d7adf8b98034 + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: '' - 976d9e4c-e7fb-4e3c-837e-a494bd7df6cb: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ab23bb84-cdc6-43db-884d-e35ad3f09349: !Template answer_choices: null - id: 976d9e4c-e7fb-4e3c-837e-a494bd7df6cb + id: ab23bb84-cdc6-43db-884d-e35ad3f09349 jinja: 'Translate Czech into English: {{translation["cs"]}} ||| {{translation["en"]}}' @@ -200,9 +197,38 @@ templates: name: !!python/tuple - translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - a9635589-6b1e-4735-8afb-6c8c5a6212f7: !Template + af36aa55-d8fc-424c-8afb-bcc920d6dec2: !Template + answer_choices: null + id: af36aa55-d8fc-424c-8afb-bcc920d6dec2 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-source+target + reference: GPT-3 paper + af4ff1b8-3a72-4cdb-a20d-0d196c36dc53: !Template + answer_choices: null + id: af4ff1b8-3a72-4cdb-a20d-0d196c36dc53 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' + d48f1328-4613-48e7-8c0a-8b0158540523: !Template answer_choices: null - id: a9635589-6b1e-4735-8afb-6c8c5a6212f7 + id: d48f1328-4613-48e7-8c0a-8b0158540523 jinja: 'If the original version says: {{translation["en"]}}; then the Czech version should say: @@ -215,80 +241,81 @@ templates: name: !!python/tuple - version-en-cs-target reference: '' - acca0487-3ff6-4572-b07f-7d740bb8c357: !Template + d5809770-ad3f-444f-8c6b-1a95ebb2d0fc: !Template answer_choices: null - id: acca0487-3ff6-4572-b07f-7d740bb8c357 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: d5809770-ad3f-444f-8c6b-1a95ebb2d0fc + jinja: 'What is the English translation of : {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-src+starget - reference: '' - b1366146-ea0d-4230-aa5d-bbbeb991977b: !Template + - gpt-3-cs-en-target + reference: GPT-3 paper + e1fcc2e4-b539-494e-aa0b-943a99d31552: !Template answer_choices: null - id: b1366146-ea0d-4230-aa5d-bbbeb991977b - jinja: 'Translate into Czech: {{translation["en"]}} + id: e1fcc2e4-b539-494e-aa0b-943a99d31552 + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target + - translate_as_cs-en-source+target reference: '' - b37f5d69-409f-491f-9c6e-0963caf0088a: !Template + e78c6092-63fd-453f-822c-0c975b58443f: !Template answer_choices: null - id: b37f5d69-409f-491f-9c6e-0963caf0088a - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: e78c6092-63fd-453f-822c-0c975b58443f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-src+target + - a_good_translation-en-cs-source+target reference: '' - cad8d76f-721d-40a4-a927-d6a4c42a7904: !Template + ebd16829-4538-4bd2-83c2-bbd9cc2ef967: !Template answer_choices: null - id: cad8d76f-721d-40a4-a927-d6a4c42a7904 - jinja: 'Translate into English: {{translation["cs"]}} + id: ebd16829-4538-4bd2-83c2-bbd9cc2ef967 + jinja: '{{translation["cs"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target - reference: '' - e2511ab0-a61a-4502-9126-7eb3afa9f8d6: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ebebf07e-6312-4b7b-ae7b-037723f5fba5: !Template answer_choices: null - id: e2511ab0-a61a-4502-9126-7eb3afa9f8d6 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: ebebf07e-6312-4b7b-ae7b-037723f5fba5 + jinja: 'Translate into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target + - translate-en-cs-target reference: '' - f701fee7-a397-4410-a7a9-1e3616de9313: !Template + ec94fdd4-30a6-430e-8886-94bc636c9224: !Template answer_choices: null - id: f701fee7-a397-4410-a7a9-1e3616de9313 - jinja: 'How do you say {{translation["cs"]}} in English? + id: ec94fdd4-30a6-430e-8886-94bc636c9224 + jinja: '{{translation["cs"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -297,12 +324,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - translate_as_cs-en-target reference: '' - fbb5e375-f3f0-4701-bdcb-c4587ea39d08: !Template + ece995e4-b66d-4fab-a9db-7b63b927ae25: !Template answer_choices: null - id: fbb5e375-f3f0-4701-bdcb-c4587ea39d08 - jinja: ' {{translation["en"]}} translates into Czech as: + id: ece995e4-b66d-4fab-a9db-7b63b927ae25 + jinja: '{{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -311,5 +338,34 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f11baf07-8675-4e85-8c0d-1b4192533f75: !Template + answer_choices: null + id: f11baf07-8675-4e85-8c0d-1b4192533f75 + jinja: 'Translate into English: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-cs-en-target + reference: '' + fe81171c-916e-4250-8775-c6b7834df0ca: !Template + answer_choices: null + id: fe81171c-916e-4250-8775-c6b7834df0ca + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-target reference: '' diff --git a/promptsource/templates/wmt15/de-en/templates.yaml b/promptsource/templates/wmt15/de-en/templates.yaml index 7548b0192..122300490 100644 --- a/promptsource/templates/wmt15/de-en/templates.yaml +++ b/promptsource/templates/wmt15/de-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt15 subset: de-en templates: - 01ab38fd-50c7-4670-9c02-7c7401e0418a: !Template + 0060361d-d7be-4181-ad60-4bf3305bd83f: !Template answer_choices: null - id: 01ab38fd-50c7-4670-9c02-7c7401e0418a - jinja: '{{translation["de"]}} translates into English as: + id: 0060361d-d7be-4181-ad60-4bf3305bd83f + jinja: 'How do you say {{translation["de"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -13,13 +13,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - how_to_say-de-en-target reference: '' - 05195a8f-6f76-48f8-9db1-d155e76314e5: !Template + 0f535f5b-f91d-403a-b4c8-cee3fb7bb939: !Template answer_choices: null - id: 05195a8f-6f76-48f8-9db1-d155e76314e5 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 0f535f5b-f91d-403a-b4c8-cee3fb7bb939 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -28,12 +28,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - version-de-en-source+target reference: '' - 0b6024e1-cc9d-4dda-b5ba-1c840e5dda8b: !Template + 111df907-6e6c-4c28-aa9f-dd4cd8874c27: !Template answer_choices: null - id: 0b6024e1-cc9d-4dda-b5ba-1c840e5dda8b - jinja: 'What is the German translation of: {{translation["en"]}} + id: 111df907-6e6c-4c28-aa9f-dd4cd8874c27 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 14d3e540-0702-492a-be73-377b71f6f5c1: !Template + answer_choices: null + id: 14d3e540-0702-492a-be73-377b71f6f5c1 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + 26bb0ef0-b326-475a-a960-ce7feee92c53: !Template + answer_choices: null + id: 26bb0ef0-b326-475a-a960-ce7feee92c53 + jinja: 'Translate English into German: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -42,26 +71,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target + - translate-en-de-source+starget reference: '' - 0e74e533-e01e-401f-8316-81fcdd11483a: !Template + 3a180fcc-2e71-4f85-addc-84a7f8b15903: !Template answer_choices: null - id: 0e74e533-e01e-401f-8316-81fcdd11483a - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 3a180fcc-2e71-4f85-addc-84a7f8b15903 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - a_good_translation-en-de-source+target reference: '' - 1023ac76-49f6-48cb-a4aa-53965ffba794: !Template + 438285d3-cb37-46b0-8f3a-2a36b69670ca: !Template answer_choices: null - id: 1023ac76-49f6-48cb-a4aa-53965ffba794 + id: 438285d3-cb37-46b0-8f3a-2a36b69670ca jinja: 'What is the German translation of the English sentence: {{translation["en"]}} ||| {{translation["de"]}}' @@ -71,11 +100,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target - reference: '' - 140eb0aa-b6ed-45e6-bd41-fb6e6c8a9cd8: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 5cd6a591-92ca-4ad4-b8b2-c4fc041b932c: !Template answer_choices: null - id: 140eb0aa-b6ed-45e6-bd41-fb6e6c8a9cd8 + id: 5cd6a591-92ca-4ad4-b8b2-c4fc041b932c jinja: 'Given the following source text in German: {{translation["de"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -86,37 +115,37 @@ templates: name: !!python/tuple - a_good_translation-de-en-source+target reference: '' - 231e54cf-c967-4998-84cb-f5feaa2bb461: !Template + 680f17bb-ec1f-44c1-a37b-edb315f4b45a: !Template answer_choices: null - id: 231e54cf-c967-4998-84cb-f5feaa2bb461 - jinja: 'Translate English into German: {{translation["en"]}} + id: 680f17bb-ec1f-44c1-a37b-edb315f4b45a + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget + - how_to_say-en-de-target reference: '' - 2c87a41f-51dd-4802-9e88-2335a31e8651: !Template + 72ed75ea-c7af-465c-ab4b-2cfe9571a301: !Template answer_choices: null - id: 2c87a41f-51dd-4802-9e88-2335a31e8651 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 72ed75ea-c7af-465c-ab4b-2cfe9571a301 + jinja: '{{translation["de"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target - reference: '' - 3cb3d385-f0c7-4626-8f3e-754d6efcd314: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 7553ecfd-104d-4bbe-b5b3-25d2baafd22f: !Template answer_choices: null - id: 3cb3d385-f0c7-4626-8f3e-754d6efcd314 + id: 7553ecfd-104d-4bbe-b5b3-25d2baafd22f jinja: 'If the original version says: {{translation["en"]}}; then the German version should say: @@ -129,10 +158,38 @@ templates: name: !!python/tuple - version-en-de-target reference: '' - 485aa919-0e7e-46cf-b8f5-d8b294599b01: !Template + a2219a0f-c82e-44b5-bf6e-83f5eeb7a8f5: !Template answer_choices: null - id: 485aa919-0e7e-46cf-b8f5-d8b294599b01 - jinja: 'Translate German into English: {{translation["de"]}} + id: a2219a0f-c82e-44b5-bf6e-83f5eeb7a8f5 + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: GPT-3 paper + a538680f-57b1-42ab-98f9-0a88b0b795e4: !Template + answer_choices: null + id: a538680f-57b1-42ab-98f9-0a88b0b795e4 + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-source+target + reference: '' + b17eacfb-a235-450f-a813-d71cb8631c51: !Template + answer_choices: null + id: b17eacfb-a235-450f-a813-d71cb8631c51 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -141,25 +198,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 8e11df98-cad8-4236-b3e3-7abc130863cc: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + b75c39a8-ba84-4934-be04-d305170da853: !Template answer_choices: null - id: 8e11df98-cad8-4236-b3e3-7abc130863cc - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: b75c39a8-ba84-4934-be04-d305170da853 + jinja: 'Translate into English: {{translation["de"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate-de-en-target reference: '' - 9af3221d-970f-429c-a4f3-7985a53e8fc3: !Template + b9cdd3d3-6d03-456f-8c78-080c5b701cbb: !Template answer_choices: null - id: 9af3221d-970f-429c-a4f3-7985a53e8fc3 - jinja: 'How do you say {{translation["de"]}} in English? + id: b9cdd3d3-6d03-456f-8c78-080c5b701cbb + jinja: 'German: {{translation["de"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -168,11 +226,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - translate_as_de-en-source+target reference: '' - 9c93367c-d0a0-4fc7-84ed-4e0ac5361416: !Template + c139ef45-c0f6-4cdf-a83a-d14b8c998c5c: !Template answer_choices: null - id: 9c93367c-d0a0-4fc7-84ed-4e0ac5361416 + id: c139ef45-c0f6-4cdf-a83a-d14b8c998c5c jinja: 'What is the English translation of : {{translation["de"]}} ||| {{translation["en"]}}' @@ -183,11 +241,11 @@ templates: original_task: true name: !!python/tuple - gpt-3-de-en-target - reference: '' - 9cf7d339-577c-4b93-8584-912158c67080: !Template + reference: GPT-3 paper + d44cfc70-e06f-4f54-b988-56de8874d577: !Template answer_choices: null - id: 9cf7d339-577c-4b93-8584-912158c67080 - jinja: 'English: {{translation["en"]}} translates into German as: + id: d44cfc70-e06f-4f54-b988-56de8874d577 + jinja: '{{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -196,13 +254,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target - reference: '' - b2180a7e-6917-4214-9189-d0b24bb850b9: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d4fa63c0-9e6b-4202-a89e-94055094559d: !Template answer_choices: null - id: b2180a7e-6917-4214-9189-d0b24bb850b9 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: d4fa63c0-9e6b-4202-a89e-94055094559d + jinja: 'English: {{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -211,105 +268,104 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target - reference: '' - b32d4b5d-a34e-4567-a880-43209f78edb7: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + d65aa853-5fe1-4d67-8153-d87f6c94e0e5: !Template answer_choices: null - id: b32d4b5d-a34e-4567-a880-43209f78edb7 - jinja: 'Translate into English: {{translation["de"]}} + id: d65aa853-5fe1-4d67-8153-d87f6c94e0e5 + jinja: 'Translate into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - translate-en-de-target reference: '' - ca0d2e57-2fda-418c-a4b6-f2626768c103: !Template + d82f4773-0cd6-4207-8c7f-51892b525a60: !Template answer_choices: null - id: ca0d2e57-2fda-418c-a4b6-f2626768c103 - jinja: 'German: {{translation["de"]}} translates into English as: + id: d82f4773-0cd6-4207-8c7f-51892b525a60 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - version-en-de-source+target reference: '' - d01c2154-db01-4689-b59f-1329d85e527b: !Template + da648068-e6b9-4409-b7bb-1cdae32dd866: !Template answer_choices: null - id: d01c2154-db01-4689-b59f-1329d85e527b - jinja: 'Translate into German: {{translation["en"]}} + id: da648068-e6b9-4409-b7bb-1cdae32dd866 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - version-de-en-target reference: '' - dcfdd38d-c333-4480-9cf8-60136d2dfae2: !Template + ecd049ba-a758-4e18-b0c8-49cf4422cbe4: !Template answer_choices: null - id: dcfdd38d-c333-4480-9cf8-60136d2dfae2 - jinja: ' {{translation["en"]}} translates into German as: - - ||| {{translation["de"]}}' + id: ecd049ba-a758-4e18-b0c8-49cf4422cbe4 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - a_good_translation-en-de-target reference: '' - e1108e0c-02cc-449e-995c-57603d161279: !Template + f6e68891-e691-4983-936b-f45574638dfe: !Template answer_choices: null - id: e1108e0c-02cc-449e-995c-57603d161279 - jinja: 'How do you say {{translation["en"]}} in German? + id: f6e68891-e691-4983-936b-f45574638dfe + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target + - translate_as_de-en-target reference: '' - e3a7ab9a-fb95-4c93-8aba-71208450998d: !Template + fbe96c98-9c98-435d-97a5-dd231fa61989: !Template answer_choices: null - id: e3a7ab9a-fb95-4c93-8aba-71208450998d - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: fbe96c98-9c98-435d-97a5-dd231fa61989 + jinja: 'Translate German into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - f0ae334c-9ea0-46bc-b474-a988218b4e6a: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + fd433a62-f316-4944-939a-441ec1dcc4ba: !Template answer_choices: null - id: f0ae334c-9ea0-46bc-b474-a988218b4e6a - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: fd433a62-f316-4944-939a-441ec1dcc4ba + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target - reference: '' + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt15/fi-en/templates.yaml b/promptsource/templates/wmt15/fi-en/templates.yaml index 0ce712f37..a70180b80 100644 --- a/promptsource/templates/wmt15/fi-en/templates.yaml +++ b/promptsource/templates/wmt15/fi-en/templates.yaml @@ -1,36 +1,37 @@ dataset: wmt15 subset: fi-en templates: - 0d4cc6d9-733f-4c67-9f79-a6d27b1f858f: !Template + 097597be-0c94-4f70-a6e8-e01e5884a637: !Template answer_choices: null - id: 0d4cc6d9-733f-4c67-9f79-a6d27b1f858f - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: 097597be-0c94-4f70-a6e8-e01e5884a637 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-src+target + - a_good_translation-fi-en-target reference: '' - 1c3f5536-818f-475a-89e4-6c62e22d439c: !Template + 099628c4-84a9-486b-bf57-a79e0cafd840: !Template answer_choices: null - id: 1c3f5536-818f-475a-89e4-6c62e22d439c - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 099628c4-84a9-486b-bf57-a79e0cafd840 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - a_good_translation-en-fi-target reference: '' - 1cc12cec-8354-4025-817d-839ea1787e1a: !Template + 0a175591-454c-4d9e-ba4f-297c30d51980: !Template answer_choices: null - id: 1cc12cec-8354-4025-817d-839ea1787e1a + id: 0a175591-454c-4d9e-ba4f-297c30d51980 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Finnish translation is: @@ -43,24 +44,23 @@ templates: name: !!python/tuple - a_good_translation-en-fi-source+target reference: '' - 2271f128-3ae4-4704-84ee-53028f57d926: !Template + 0ce942c1-26d7-4b3c-b9ff-24ff2596a573: !Template answer_choices: null - id: 2271f128-3ae4-4704-84ee-53028f57d926 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 0ce942c1-26d7-4b3c-b9ff-24ff2596a573 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target - reference: '' - 3cf8e8eb-c4aa-4211-8779-97a4f133546e: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 173fa2fc-3e38-483e-9674-f6d7d50c63d1: !Template answer_choices: null - id: 3cf8e8eb-c4aa-4211-8779-97a4f133546e + id: 173fa2fc-3e38-483e-9674-f6d7d50c63d1 jinja: 'If the original version says: {{translation["fi"]}}; then the English version should say: @@ -73,23 +73,23 @@ templates: name: !!python/tuple - version-fi-en-target reference: '' - 4e34023a-b5fb-428b-979d-374126074714: !Template + 3a8a540f-255c-4e79-8179-5089affbf69b: !Template answer_choices: null - id: 4e34023a-b5fb-428b-979d-374126074714 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 3a8a540f-255c-4e79-8179-5089affbf69b + jinja: 'What is the Finnish translation of: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target - reference: '' - 4fe7ff1c-d803-45da-8fb7-ada4f3c70fe2: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 3ddc5563-da90-4196-9292-86a709fdfd75: !Template answer_choices: null - id: 4fe7ff1c-d803-45da-8fb7-ada4f3c70fe2 + id: 3ddc5563-da90-4196-9292-86a709fdfd75 jinja: 'Finnish: {{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' @@ -101,23 +101,51 @@ templates: name: !!python/tuple - translate_as_fi-en-source+target reference: '' - 663ee2dd-d784-4fc4-a0d5-25705c000ef0: !Template + 3e5f95c4-8fe5-4dd9-960c-ff2e4b479d2b: !Template answer_choices: null - id: 663ee2dd-d784-4fc4-a0d5-25705c000ef0 - jinja: 'How do you say {{translation["fi"]}} in English? + id: 3e5f95c4-8fe5-4dd9-960c-ff2e4b479d2b + jinja: 'Translate Finnish into English: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 44d7d5db-5002-43aa-b625-5664d1d98167: !Template + answer_choices: null + id: 44d7d5db-5002-43aa-b625-5664d1d98167 + jinja: 'How do you say {{translation["en"]}} in Finnish? + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-fi-target reference: '' - 77757cf4-6946-414a-967d-136ce9a17b50: !Template + 57e1762c-122b-4f3b-8f8e-2be3c79d0f92: !Template answer_choices: null - id: 77757cf4-6946-414a-967d-136ce9a17b50 + id: 57e1762c-122b-4f3b-8f8e-2be3c79d0f92 + jinja: 'Translate English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-source+starget + reference: '' + 5ea50f65-59de-4387-8e70-36e556d2718b: !Template + answer_choices: null + id: 5ea50f65-59de-4387-8e70-36e556d2718b jinja: '{{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' @@ -129,10 +157,25 @@ templates: name: !!python/tuple - translate_as_fi-en-target reference: '' - 7a2a2a98-662e-45ea-b1ea-cced5318feb8: !Template + 625b6912-3aa8-46b7-b8f8-0d21b2fe0a23: !Template answer_choices: null - id: 7a2a2a98-662e-45ea-b1ea-cced5318feb8 - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 625b6912-3aa8-46b7-b8f8-0d21b2fe0a23 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target + reference: '' + 6795f8d5-cec3-47f4-b1bd-55e9c0690941: !Template + answer_choices: null + id: 6795f8d5-cec3-47f4-b1bd-55e9c0690941 + jinja: 'English: {{translation["en"]}} translates into Finnish as: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -141,41 +184,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - translate_as_en-fi-source+target reference: '' - 8bdabcfa-3d40-4d45-b7cb-1806570397fe: !Template + 6b240ab5-9487-43fc-bb3e-ea5325cbcb7e: !Template answer_choices: null - id: 8bdabcfa-3d40-4d45-b7cb-1806570397fe - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 6b240ab5-9487-43fc-bb3e-ea5325cbcb7e + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-src+starget + - translate_as_en-fi-target reference: '' - 95a1bcc5-fb23-4fec-b81d-93eaa955d615: !Template + 75957d57-91a0-4627-be8b-02f920b1b42f: !Template answer_choices: null - id: 95a1bcc5-fb23-4fec-b81d-93eaa955d615 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 75957d57-91a0-4627-be8b-02f920b1b42f + jinja: 'Translate into English: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - translate-fi-en-target reference: '' - 9639fd0e-1f76-446d-9c7a-0d9c3a10dca2: !Template + 7601409d-7187-4658-800b-99d827a1c782: !Template answer_choices: null - id: 9639fd0e-1f76-446d-9c7a-0d9c3a10dca2 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 7601409d-7187-4658-800b-99d827a1c782 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -184,39 +226,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 993eb10a-38d7-4880-9432-e73834b06893: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 881571bd-3405-442b-8f00-7eabbba158a4: !Template answer_choices: null - id: 993eb10a-38d7-4880-9432-e73834b06893 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 881571bd-3405-442b-8f00-7eabbba158a4 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target + - a_good_translation-fi-en-source+target reference: '' - a799d0ab-ece3-405b-a492-d0f177767f1e: !Template + 91836597-c57e-4cf5-b007-112278f7db24: !Template answer_choices: null - id: a799d0ab-ece3-405b-a492-d0f177767f1e - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 91836597-c57e-4cf5-b007-112278f7db24 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target + - how_to_say-fi-en-target reference: '' - aa6dba99-10d7-4fd0-a5c1-263de99e3759: !Template + 9d287a58-6ac5-42b4-8825-1645718a358d: !Template answer_choices: null - id: aa6dba99-10d7-4fd0-a5c1-263de99e3759 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: 9d287a58-6ac5-42b4-8825-1645718a358d + jinja: 'English: {{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + afa9f710-d1da-4a94-8ffb-dcb45567b8bf: !Template + answer_choices: null + id: afa9f710-d1da-4a94-8ffb-dcb45567b8bf + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -225,91 +281,91 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target - reference: '' - afe000ca-c5f9-4138-85ed-4d1c073f3d10: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + b2d1804c-6074-4981-afd5-443e72b53743: !Template answer_choices: null - id: afe000ca-c5f9-4138-85ed-4d1c073f3d10 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: + id: b2d1804c-6074-4981-afd5-443e72b53743 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target - reference: '' - baba0fc5-1dad-423f-8b11-90f3f06b0bb2: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ba4e4a2e-7c03-4ed0-8e3c-a125e7cd81cb: !Template answer_choices: null - id: baba0fc5-1dad-423f-8b11-90f3f06b0bb2 - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: ba4e4a2e-7c03-4ed0-8e3c-a125e7cd81cb + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target + - version-en-fi-source+target reference: '' - cf22aa75-de10-42c0-9141-4fc2953fe1b7: !Template + c441750d-f758-4626-be61-d180894648c4: !Template answer_choices: null - id: cf22aa75-de10-42c0-9141-4fc2953fe1b7 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: c441750d-f758-4626-be61-d180894648c4 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-src+target + - version-en-fi-target reference: '' - cf59bd05-215a-4dfd-9634-aa2c6c8147cc: !Template + cc6dba77-a277-4a71-b667-32636da88b88: !Template answer_choices: null - id: cf59bd05-215a-4dfd-9634-aa2c6c8147cc - jinja: 'Translate into English: {{translation["fi"]}} + id: cc6dba77-a277-4a71-b667-32636da88b88 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target - reference: '' - eb22bd68-0c81-4389-9370-64f8cdd0e037: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d41a01f1-66bf-4d54-9542-2e9cf1c14cf4: !Template answer_choices: null - id: eb22bd68-0c81-4389-9370-64f8cdd0e037 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: d41a01f1-66bf-4d54-9542-2e9cf1c14cf4 + jinja: 'Translate into Finnish: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target + - translate-en-fi-target reference: '' - f618b9c0-6c74-450d-bf95-d711ad40f457: !Template + fc2489d5-89d8-41bc-ab95-4646d533d25a: !Template answer_choices: null - id: f618b9c0-6c74-450d-bf95-d711ad40f457 - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: fc2489d5-89d8-41bc-ab95-4646d533d25a + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target - reference: '' + - gpt-3-fi-en-target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt15/fr-en/templates.yaml b/promptsource/templates/wmt15/fr-en/templates.yaml index accda865e..9d177bcb1 100644 --- a/promptsource/templates/wmt15/fr-en/templates.yaml +++ b/promptsource/templates/wmt15/fr-en/templates.yaml @@ -1,52 +1,53 @@ dataset: wmt15 subset: fr-en templates: - 086d3670-4e19-488d-a12a-3c635b0b0675: !Template + 088f8a76-818e-461b-874f-c358123a2c7a: !Template answer_choices: null - id: 086d3670-4e19-488d-a12a-3c635b0b0675 - jinja: 'Translate French into English: {{translation["fr"]}} + id: 088f8a76-818e-461b-874f-c358123a2c7a + jinja: 'Translate English into French: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 0c485a59-4f19-43ea-a228-dd9daa95581a: !Template + - translate-en-fr-source+starget + reference: '' + 0d1ef213-45e0-4fa3-aa60-56f06ab20258: !Template answer_choices: null - id: 0c485a59-4f19-43ea-a228-dd9daa95581a - jinja: 'English: {{translation["en"]}} translates into French as: + id: 0d1ef213-45e0-4fa3-aa60-56f06ab20258 + jinja: 'If the original version says: {{translation["en"]}}; then the French version + should say: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fr-source+target + - version-en-fr-target reference: '' - 0d317972-73b8-4f8d-9c9d-e3fcdfa39781: !Template + 11ebba9c-de47-4415-8016-6bf84a7566eb: !Template answer_choices: null - id: 0d317972-73b8-4f8d-9c9d-e3fcdfa39781 - jinja: 'Translate into English: {{translation["fr"]}} + id: 11ebba9c-de47-4415-8016-6bf84a7566eb + jinja: '{{translation["fr"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-en-target - reference: '' - 10b1b2d8-4ade-4545-b9cd-0bdc20f83b17: !Template + - xglm-fr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 177bb7d6-4b09-4ba0-9717-49ca57ee9442: !Template answer_choices: null - id: 10b1b2d8-4ade-4545-b9cd-0bdc20f83b17 - jinja: 'Translate English into French: {{translation["en"]}} + id: 177bb7d6-4b09-4ba0-9717-49ca57ee9442 + jinja: 'What is the French translation of the English sentence: {{translation["en"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -55,69 +56,68 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fr-src+starget - reference: '' - 13b8c6a0-b968-43ed-bdf5-3ca2acc98c40: !Template + - gpt-3-en-fr-source+target + reference: GPT 3 paper + 1e881c1f-f418-471f-908b-b9ba7a1a6ecd: !Template answer_choices: null - id: 13b8c6a0-b968-43ed-bdf5-3ca2acc98c40 - jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation - is: + id: 1e881c1f-f418-471f-908b-b9ba7a1a6ecd + jinja: 'Translate French into English: {{translation["fr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-en-target - reference: '' - 15e9c7fb-31b3-44f2-b56c-3dac3aa921d5: !Template + - translate-fr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1ecf81db-5538-4d8b-9d16-5384b972143f: !Template answer_choices: null - id: 15e9c7fb-31b3-44f2-b56c-3dac3aa921d5 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good French translation is: + id: 1ecf81db-5538-4d8b-9d16-5384b972143f + jinja: '{{translation["fr"]}} translates into English as: - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fr-source+target + - translate_as_fr-en-target reference: '' - 1bfca273-4e3b-42dc-ba5d-11a2823d3a5a: !Template + 214598b5-16dc-4c42-8e55-e429ccd11c4d: !Template answer_choices: null - id: 1bfca273-4e3b-42dc-ba5d-11a2823d3a5a - jinja: 'Given the following passage: {{translation["en"]}} , a good French translation - is: ||| {{translation["fr"]}}' + id: 214598b5-16dc-4c42-8e55-e429ccd11c4d + jinja: 'Translate into French: {{translation["en"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fr-target + - translate-en-fr-target reference: '' - 226f7037-a37d-4154-a5f5-d613afc3ec76: !Template + 342fb592-d2e8-40a8-b053-f3cae6ceb2c7: !Template answer_choices: null - id: 226f7037-a37d-4154-a5f5-d613afc3ec76 - jinja: 'What is the French translation of: {{translation["en"]}} + id: 342fb592-d2e8-40a8-b053-f3cae6ceb2c7 + jinja: 'English: {{translation["en"]}} = French: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fr-target - reference: '' - 405e991d-ec19-4a49-a45e-a2ebe98044a0: !Template + - xglm-en-fr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 3891aaee-2a70-40b7-9e7c-2ad6d43e5fa1: !Template answer_choices: null - id: 405e991d-ec19-4a49-a45e-a2ebe98044a0 - jinja: 'What is the English translation of : {{translation["fr"]}} + id: 3891aaee-2a70-40b7-9e7c-2ad6d43e5fa1 + jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -126,11 +126,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-en-target - reference: '' - 40ed2bb6-4007-48bd-bf3f-569784505916: !Template + - gpt-3-fr-en-source+target + reference: GPT-3 paper + 3ccaebb0-0857-4f4c-a21c-90e4c2f19797: !Template + answer_choices: null + id: 3ccaebb0-0857-4f4c-a21c-90e4c2f19797 + jinja: 'French: {{translation["fr"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-fr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 42003670-59f2-4113-bf67-f36d00972dca: !Template answer_choices: null - id: 40ed2bb6-4007-48bd-bf3f-569784505916 + id: 42003670-59f2-4113-bf67-f36d00972dca jinja: 'If the French version says: {{translation["fr"]}}; then the English version should say: @@ -143,10 +157,10 @@ templates: name: !!python/tuple - version-fr-en-source+target reference: '' - 432c00d7-74c8-415d-b931-a7cc05211a7a: !Template + 59efc156-32fb-40d4-b639-5f02233137e0: !Template answer_choices: null - id: 432c00d7-74c8-415d-b931-a7cc05211a7a - jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} + id: 59efc156-32fb-40d4-b639-5f02233137e0 + jinja: 'What is the English translation of : {{translation["fr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -155,12 +169,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-en-src+target - reference: '' - 5b1b257b-457d-4a7e-9a2a-ef6924ba6b01: !Template + - gpt-3-fr-en-target + reference: GPT-3 paper + 608dddb5-9544-4d1a-9a3a-6e2179ed53f5: !Template answer_choices: null - id: 5b1b257b-457d-4a7e-9a2a-ef6924ba6b01 - jinja: 'Translate into French: {{translation["en"]}} + id: 608dddb5-9544-4d1a-9a3a-6e2179ed53f5 + jinja: 'What is the French translation of: {{translation["en"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -169,69 +183,97 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fr-target + - gpt-3-en-fr-target + reference: GPT-3 paper + 6961a384-b948-47e4-9197-a7c8e46b5e7d: !Template + answer_choices: null + id: 6961a384-b948-47e4-9197-a7c8e46b5e7d + jinja: 'If the English version says: {{translation["en"]}}; then the French version + should say: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-fr-source+target reference: '' - 8ce8cd9f-8c76-4559-8a40-9b4b6efc1408: !Template + 6d5bcb5d-8f39-4543-86ae-23534bc79647: !Template answer_choices: null - id: 8ce8cd9f-8c76-4559-8a40-9b4b6efc1408 - jinja: 'How do you say {{translation["fr"]}} in English? + id: 6d5bcb5d-8f39-4543-86ae-23534bc79647 + jinja: 'English: {{translation["en"]}} translates into French as: - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fr-en-target + - translate_as_en-fr-source+target reference: '' - 8d262474-b465-45b2-b39a-0f1f95b0a9c0: !Template + 735d779e-6cea-4453-b000-78b2c7f6c01e: !Template answer_choices: null - id: 8d262474-b465-45b2-b39a-0f1f95b0a9c0 - jinja: 'French: {{translation["fr"]}} translates into English as: + id: 735d779e-6cea-4453-b000-78b2c7f6c01e + jinja: 'Translate into English: {{translation["fr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-en-source+target + - translate-fr-en-target reference: '' - 9180b8ec-ec77-47cf-a930-3832691cb210: !Template + 7456ba86-e183-42c5-92e7-00280adbe74c: !Template answer_choices: null - id: 9180b8ec-ec77-47cf-a930-3832691cb210 - jinja: 'If the original version says: {{translation["en"]}}; then the French version - should say: + id: 7456ba86-e183-42c5-92e7-00280adbe74c + jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation + is: - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fr-target + - a_good_translation-fr-en-target reference: '' - a0bc23d9-664b-45b8-9ccd-b3c16f70b28c: !Template + 74ada6ba-9a9a-4960-aaf6-7633b9cb6f28: !Template answer_choices: null - id: a0bc23d9-664b-45b8-9ccd-b3c16f70b28c - jinja: 'If the original version says: {{translation["fr"]}}; then the English - version should say: + id: 74ada6ba-9a9a-4960-aaf6-7633b9cb6f28 + jinja: 'French: {{translation["fr"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fr-en-target + - translate_as_fr-en-source+target reference: '' - a38e45aa-12ce-4c64-a2d9-76f184aec707: !Template + 8ea8f712-3c4c-49f3-ad72-f247ba0d9953: !Template answer_choices: null - id: a38e45aa-12ce-4c64-a2d9-76f184aec707 + id: 8ea8f712-3c4c-49f3-ad72-f247ba0d9953 + jinja: '{{translation["en"]}} = French: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-fr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9dd64acc-81cc-4594-a132-41d65caadf77: !Template + answer_choices: null + id: 9dd64acc-81cc-4594-a132-41d65caadf77 jinja: 'How do you say {{translation["en"]}} in French? ||| {{translation["fr"]}}' @@ -243,66 +285,67 @@ templates: name: !!python/tuple - how_to_say-en-fr-target reference: '' - a6d3c7fe-e155-4677-bbf6-30566f29faf9: !Template + a4e97bcf-530d-4244-a5d1-a7f0475ce671: !Template answer_choices: null - id: a6d3c7fe-e155-4677-bbf6-30566f29faf9 - jinja: 'What is the French translation of the English sentence: {{translation["en"]}} + id: a4e97bcf-530d-4244-a5d1-a7f0475ce671 + jinja: 'How do you say {{translation["fr"]}} in English? - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fr-src+target + - how_to_say-fr-en-target reference: '' - b2b616a0-b613-4fa3-86b6-efca11803f9c: !Template + aa52033d-58f0-466d-b050-3cf8b5fc98c6: !Template answer_choices: null - id: b2b616a0-b613-4fa3-86b6-efca11803f9c - jinja: 'Given the following source text in French: {{translation["fr"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: aa52033d-58f0-466d-b050-3cf8b5fc98c6 + jinja: 'Given the following passage: {{translation["en"]}} , a good French translation + is: ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-en-source+target + - a_good_translation-en-fr-target reference: '' - bb9cc49c-adcd-487b-8fac-49847fa4c680: !Template + b392064e-614f-4df4-b2ca-0637d853c72e: !Template answer_choices: null - id: bb9cc49c-adcd-487b-8fac-49847fa4c680 - jinja: '{{translation["fr"]}} translates into English as: + id: b392064e-614f-4df4-b2ca-0637d853c72e + jinja: 'If the original version says: {{translation["fr"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-en-target + - version-fr-en-target reference: '' - c205c9cc-f09f-4d55-ae7b-c81337ee95e1: !Template + b3e92821-842c-4ae9-be87-12c805f2cbc3: !Template answer_choices: null - id: c205c9cc-f09f-4d55-ae7b-c81337ee95e1 - jinja: 'If the English version says: {{translation["en"]}}; then the French version - should say: + id: b3e92821-842c-4ae9-be87-12c805f2cbc3 + jinja: ' {{translation["en"]}} translates into French as: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fr-source+target + - translate_as_en-fr-target reference: '' - ff2ca060-1384-4836-b0cd-6c48c84bf9ae: !Template + cc0e7dfc-5308-4560-bcfe-eb95511e1868: !Template answer_choices: null - id: ff2ca060-1384-4836-b0cd-6c48c84bf9ae - jinja: ' {{translation["en"]}} translates into French as: + id: cc0e7dfc-5308-4560-bcfe-eb95511e1868 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good French translation is: ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -311,5 +354,18 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fr-target + - a_good_translation-en-fr-source+target + reference: '' + fdebd9e6-ac4a-4679-b725-26c3286c5ef1: !Template + answer_choices: null + id: fdebd9e6-ac4a-4679-b725-26c3286c5ef1 + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-fr-en-source+target reference: '' diff --git a/promptsource/templates/wmt15/ru-en/templates.yaml b/promptsource/templates/wmt15/ru-en/templates.yaml index 358b58b61..bc4a975bf 100644 --- a/promptsource/templates/wmt15/ru-en/templates.yaml +++ b/promptsource/templates/wmt15/ru-en/templates.yaml @@ -1,94 +1,96 @@ dataset: wmt15 subset: ru-en templates: - 1318da6d-555b-4c5d-aa31-5f8b4a820a8d: !Template + 03df7ac2-16f5-44c8-87c2-b5223a4c94e9: !Template answer_choices: null - id: 1318da6d-555b-4c5d-aa31-5f8b4a820a8d - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 03df7ac2-16f5-44c8-87c2-b5223a4c94e9 + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: '' - 155a3c3c-ef8e-4956-93cb-4d91231e3b9c: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 089ff29c-4e66-45f8-bf87-95aa1768482b: !Template answer_choices: null - id: 155a3c3c-ef8e-4956-93cb-4d91231e3b9c - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 089ff29c-4e66-45f8-bf87-95aa1768482b + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target - reference: '' - 41902388-737e-4a3c-9bdf-efc470895d14: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 0dd5c911-ec29-4000-a486-ecf5f4057fc4: !Template answer_choices: null - id: 41902388-737e-4a3c-9bdf-efc470895d14 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 0dd5c911-ec29-4000-a486-ecf5f4057fc4 + jinja: 'English: {{translation["en"]}} = Russian: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget - reference: '' - 48c610c6-72f5-4665-9822-addf0bd0037d: !Template + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 193b2702-e0c2-4189-b7b4-3ab2417f2a07: !Template answer_choices: null - id: 48c610c6-72f5-4665-9822-addf0bd0037d - jinja: 'Translate into English: {{translation["ru"]}} + id: 193b2702-e0c2-4189-b7b4-3ab2417f2a07 + jinja: 'How do you say {{translation["ru"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - how_to_say-ru-en-target reference: '' - 67dcb08c-c4eb-4da7-8249-0d51c04864d3: !Template + 19850a56-1fe3-4e43-a78d-3a001a0ca1eb: !Template answer_choices: null - id: 67dcb08c-c4eb-4da7-8249-0d51c04864d3 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 19850a56-1fe3-4e43-a78d-3a001a0ca1eb + jinja: 'English: {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 6f1eeb41-0174-4d9a-a5da-942c4017d352: !Template + - translate_as_en-ru-source+target + reference: '' + 1f9b14d6-974c-4f13-9ee3-fc6ea0978ff4: !Template answer_choices: null - id: 6f1eeb41-0174-4d9a-a5da-942c4017d352 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 1f9b14d6-974c-4f13-9ee3-fc6ea0978ff4 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - a_good_translation-ru-en-target reference: '' - 6f239ff6-5f43-4951-959f-1cd39359cc60: !Template + 24a43c12-fbc5-48f1-afe1-099a978a0c6a: !Template answer_choices: null - id: 6f239ff6-5f43-4951-959f-1cd39359cc60 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 24a43c12-fbc5-48f1-afe1-099a978a0c6a + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -97,12 +99,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - version-en-ru-source+target reference: '' - 73821062-32c8-4589-8c35-274f75ff1bc1: !Template + 26881ddf-0d5d-42ef-ad3c-52bfd8cf861d: !Template answer_choices: null - id: 73821062-32c8-4589-8c35-274f75ff1bc1 - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 26881ddf-0d5d-42ef-ad3c-52bfd8cf861d + jinja: '{{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -111,42 +113,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target - reference: '' - 75d10f69-f208-4795-bc0e-032c6d5d1744: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 271ddd3e-2f20-41b9-8ca3-de0ba9337214: !Template answer_choices: null - id: 75d10f69-f208-4795-bc0e-032c6d5d1744 - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 271ddd3e-2f20-41b9-8ca3-de0ba9337214 + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' - 77cc43eb-c7df-4618-9707-36bd04038787: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 2e262e52-3377-4764-bba5-f2f7d2a90bbd: !Template answer_choices: null - id: 77cc43eb-c7df-4618-9707-36bd04038787 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 2e262e52-3377-4764-bba5-f2f7d2a90bbd + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - version-en-ru-target reference: '' - a2ac1716-1abb-467e-9bee-cff01bf1beab: !Template + 3cbb11b5-a323-411d-b5c9-2ecf951f67b2: !Template answer_choices: null - id: a2ac1716-1abb-467e-9bee-cff01bf1beab - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 3cbb11b5-a323-411d-b5c9-2ecf951f67b2 + jinja: 'Translate into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -155,12 +156,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - translate-en-ru-target reference: '' - afc46ae1-90f4-4db0-a576-3f5804071f46: !Template + 54c91c79-b709-40aa-85d6-68999f351fb3: !Template answer_choices: null - id: afc46ae1-90f4-4db0-a576-3f5804071f46 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 54c91c79-b709-40aa-85d6-68999f351fb3 + jinja: 'Translate into English: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -169,26 +170,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target + - translate-ru-en-target reference: '' - c3be2d8a-23db-46d5-9603-06cb7e6c9e80: !Template + 69a4c8a9-8d0c-4900-a176-c2c03893cba0: !Template answer_choices: null - id: c3be2d8a-23db-46d5-9603-06cb7e6c9e80 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 69a4c8a9-8d0c-4900-a176-c2c03893cba0 + jinja: 'Translate Russian into English: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target - reference: '' - c9d135b5-0740-4a36-a8b3-f45a14380d32: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 7443b4b0-34e6-4faf-8e3b-119383aba078: !Template answer_choices: null - id: c9d135b5-0740-4a36-a8b3-f45a14380d32 + id: 7443b4b0-34e6-4faf-8e3b-119383aba078 jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version should say: @@ -201,53 +201,51 @@ templates: name: !!python/tuple - version-ru-en-source+target reference: '' - d8a8472e-d8b2-41e0-bbf8-1c906e63c6cd: !Template + 7828e715-1e1c-40b3-aa38-88b2f1281b18: !Template answer_choices: null - id: d8a8472e-d8b2-41e0-bbf8-1c906e63c6cd - jinja: '{{translation["ru"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 7828e715-1e1c-40b3-aa38-88b2f1281b18 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - a_good_translation-en-ru-target reference: '' - e0c37fa9-2597-4482-aad4-578369694cc4: !Template + 89c69c2e-df37-46b2-8679-09e3ab676dfd: !Template answer_choices: null - id: e0c37fa9-2597-4482-aad4-578369694cc4 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 89c69c2e-df37-46b2-8679-09e3ab676dfd + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - translate_as_ru-en-source+target reference: '' - e2581600-24bd-4df3-a326-df1d285b7925: !Template + abe216ca-9422-4a33-ac6f-10901b696cfa: !Template answer_choices: null - id: e2581600-24bd-4df3-a326-df1d285b7925 - jinja: 'What is the English translation of : {{translation["ru"]}} + id: abe216ca-9422-4a33-ac6f-10901b696cfa + jinja: 'Translate English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target + - translate-en-ru-source+starget reference: '' - ee837cf8-ae01-46de-8157-be27d9d14515: !Template + ba114e8a-d16c-406d-a67f-e0add4293708: !Template answer_choices: null - id: ee837cf8-ae01-46de-8157-be27d9d14515 - jinja: 'Translate into Russian: {{translation["en"]}} + id: ba114e8a-d16c-406d-a67f-e0add4293708 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -256,25 +254,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - gpt-3-en-ru-source+target + reference: GPT 3 paper + c0e0b76b-ad5c-4d9b-a76d-8845c10f72ae: !Template + answer_choices: null + id: c0e0b76b-ad5c-4d9b-a76d-8845c10f72ae + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target reference: '' - f3bec842-663c-4300-b496-616216ceff6a: !Template + c414fb01-ded7-46a3-a86c-d313f04e449b: !Template answer_choices: null - id: f3bec842-663c-4300-b496-616216ceff6a - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: c414fb01-ded7-46a3-a86c-d313f04e449b + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - a_good_translation-ru-en-source+target reference: '' - f6409e35-9e64-4dea-a637-648bf33cb8ac: !Template + c84f5647-6d22-4e23-b7a0-13fbc5a3345f: !Template answer_choices: null - id: f6409e35-9e64-4dea-a637-648bf33cb8ac - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: c84f5647-6d22-4e23-b7a0-13fbc5a3345f + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-ru-target + reference: GPT-3 paper + cf4680f0-7c08-4fcb-b6ea-ac03b143894c: !Template + answer_choices: null + id: cf4680f0-7c08-4fcb-b6ea-ac03b143894c + jinja: 'How do you say {{translation["en"]}} in Russian? ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -283,12 +309,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - how_to_say-en-ru-target reference: '' - fbe341d6-74a0-4af4-b5a4-e9c1f81ff5c7: !Template + db532de1-be7e-479b-965b-ab2f6a34f01c: !Template answer_choices: null - id: fbe341d6-74a0-4af4-b5a4-e9c1f81ff5c7 - jinja: 'How do you say {{translation["ru"]}} in English? + id: db532de1-be7e-479b-965b-ab2f6a34f01c + jinja: '{{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -297,12 +323,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate_as_ru-en-target reference: '' - fd0c2f56-55b4-42e9-9867-be1af5b2584c: !Template + ee253aa4-b21e-47fc-87e1-d033b9d4a5f9: !Template answer_choices: null - id: fd0c2f56-55b4-42e9-9867-be1af5b2584c - jinja: ' {{translation["en"]}} translates into Russian as: + id: ee253aa4-b21e-47fc-87e1-d033b9d4a5f9 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ru-en-source+target + reference: GPT-3 paper + f21d7b09-21cd-426d-b323-cccb6a29db04: !Template + answer_choices: null + id: f21d7b09-21cd-426d-b323-cccb6a29db04 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -311,5 +352,20 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - a_good_translation-en-ru-source+target + reference: '' + fd9c5bcf-5c69-4af6-905e-ae08247b4278: !Template + answer_choices: null + id: fd9c5bcf-5c69-4af6-905e-ae08247b4278 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-target reference: '' diff --git a/promptsource/templates/wmt16/cs-en/templates.yaml b/promptsource/templates/wmt16/cs-en/templates.yaml index 3062f18a2..bf5d9c0be 100644 --- a/promptsource/templates/wmt16/cs-en/templates.yaml +++ b/promptsource/templates/wmt16/cs-en/templates.yaml @@ -1,68 +1,65 @@ dataset: wmt16 subset: cs-en templates: - 0892dd27-5006-4066-b388-ddc21920ca1b: !Template + 0a23584b-3021-44e5-a3db-6f47ce2d521e: !Template answer_choices: null - id: 0892dd27-5006-4066-b388-ddc21920ca1b - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 0a23584b-3021-44e5-a3db-6f47ce2d521e + jinja: '{{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-src+starget + - translate_as_cs-en-target reference: '' - 10cb3276-9c00-471f-97ad-35ecadaff00b: !Template + 0b2c143d-d2e5-4657-9848-e7ed35600074: !Template answer_choices: null - id: 10cb3276-9c00-471f-97ad-35ecadaff00b - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: 0b2c143d-d2e5-4657-9848-e7ed35600074 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target - reference: '' - 2098c870-299b-44ab-a7fa-44b347cb7294: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + 176357b9-af45-4380-89fb-e53b82198c76: !Template answer_choices: null - id: 2098c870-299b-44ab-a7fa-44b347cb7294 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: 176357b9-af45-4380-89fb-e53b82198c76 + jinja: '{{translation["en"]}} = Czech: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target - reference: '' - 24d73443-a15a-42cc-99a3-7fac8f4bc67f: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 188e0622-6a3a-49ff-89c9-7c3b93cbabfc: !Template answer_choices: null - id: 24d73443-a15a-42cc-99a3-7fac8f4bc67f - jinja: 'English: {{translation["en"]}} translates into Czech as: - - ||| {{translation["cs"]}}' + id: 188e0622-6a3a-49ff-89c9-7c3b93cbabfc + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - a_good_translation-en-cs-target reference: '' - 258a4b04-aea6-4ea2-9e05-68252a71a9ab: !Template + 18d8b05a-25db-45ab-b383-5ad387e6c3cb: !Template answer_choices: null - id: 258a4b04-aea6-4ea2-9e05-68252a71a9ab - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + id: 18d8b05a-25db-45ab-b383-5ad387e6c3cb + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version should say: ||| {{translation["cs"]}}' @@ -72,26 +69,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target + - version-en-cs-source+target reference: '' - 25d84bda-fb1d-4239-8fc0-64f6088a1e01: !Template + 1b264c9e-1326-4754-ba69-ce63173523ca: !Template answer_choices: null - id: 25d84bda-fb1d-4239-8fc0-64f6088a1e01 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: 1b264c9e-1326-4754-ba69-ce63173523ca + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 2976ce81-af77-4f32-b74b-4a5b889660ee: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + 4d9cd1b4-7af8-46f8-a9ad-d5d888e97c19: !Template answer_choices: null - id: 2976ce81-af77-4f32-b74b-4a5b889660ee - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 4d9cd1b4-7af8-46f8-a9ad-d5d888e97c19 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-cs-en-source+target + reference: '' + 6b0c4291-aa98-4960-9954-e43d003ac005: !Template + answer_choices: null + id: 6b0c4291-aa98-4960-9954-e43d003ac005 + jinja: 'What is the Czech translation of: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -100,25 +110,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-src+target - reference: '' - 2cccae35-0560-4b41-9ff4-07e59cef3bee: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 7bdf903f-6dd7-49fa-bffd-f493130dacc5: !Template answer_choices: null - id: 2cccae35-0560-4b41-9ff4-07e59cef3bee - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 7bdf903f-6dd7-49fa-bffd-f493130dacc5 + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_cs-en-source+target reference: '' - 306d35e0-e9ea-45dc-abf8-fed4df11fdc9: !Template + 7cc9b4f1-429c-4435-b231-4a92f27dc839: !Template answer_choices: null - id: 306d35e0-e9ea-45dc-abf8-fed4df11fdc9 + id: 7cc9b4f1-429c-4435-b231-4a92f27dc839 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Czech translation is: @@ -131,52 +141,53 @@ templates: name: !!python/tuple - a_good_translation-en-cs-source+target reference: '' - 3c095e27-6f4c-432d-b633-157a28e70a82: !Template + 7de01c31-1280-4ac5-b880-77bc539538fe: !Template answer_choices: null - id: 3c095e27-6f4c-432d-b633-157a28e70a82 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: 7de01c31-1280-4ac5-b880-77bc539538fe + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target + - how_to_say-en-cs-target reference: '' - 436eefe7-030f-4d9d-9791-c21c59456469: !Template + 83b65581-14ea-4d11-b898-5289ab641dfa: !Template answer_choices: null - id: 436eefe7-030f-4d9d-9791-c21c59456469 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 83b65581-14ea-4d11-b898-5289ab641dfa + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-src+target + - version-cs-en-target reference: '' - 49994c64-6eb3-4874-8f90-da42f692a0a1: !Template + 9d85919c-6750-4c8f-b1d6-c4575c1f9333: !Template answer_choices: null - id: 49994c64-6eb3-4874-8f90-da42f692a0a1 - jinja: ' {{translation["en"]}} translates into Czech as: + id: 9d85919c-6750-4c8f-b1d6-c4575c1f9333 + jinja: 'Translate English into Czech: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - translate-en-cs-source+starget reference: '' - 70ff8130-2515-4e14-a18c-d6eea4118893: !Template + a565febb-6876-4f4c-96cc-2239ca19cd9d: !Template answer_choices: null - id: 70ff8130-2515-4e14-a18c-d6eea4118893 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: a565febb-6876-4f4c-96cc-2239ca19cd9d + jinja: 'Translate Czech into English: {{translation["cs"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -185,51 +196,56 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target - reference: '' - 8a7b8660-b64c-4207-a224-c2bb3c29e36f: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + a59f9aa4-48c4-420d-98d2-91739afcb2fe: !Template answer_choices: null - id: 8a7b8660-b64c-4207-a224-c2bb3c29e36f - jinja: 'Translate into Czech: {{translation["en"]}} + id: a59f9aa4-48c4-420d-98d2-91739afcb2fe + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target + - version-cs-en-source+target reference: '' - 9a400489-d8b6-4147-9595-538f63bbee8a: !Template + accb80ac-3784-450b-88e7-afd3c8b161d7: !Template answer_choices: null - id: 9a400489-d8b6-4147-9595-538f63bbee8a - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: accb80ac-3784-450b-88e7-afd3c8b161d7 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - a_good_translation-cs-en-target reference: '' - a3f15b8e-5398-42ba-aae9-13b7fd9e2285: !Template + b19dedc6-20ef-47d3-b755-fb7bf0e11001: !Template answer_choices: null - id: a3f15b8e-5398-42ba-aae9-13b7fd9e2285 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: b19dedc6-20ef-47d3-b755-fb7bf0e11001 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - version-en-cs-target reference: '' - aac0fa41-9ea0-4e27-b2c0-384515f23fda: !Template + b4c94927-68ae-4ef4-bbd1-1c2c986cb504: !Template answer_choices: null - id: aac0fa41-9ea0-4e27-b2c0-384515f23fda + id: b4c94927-68ae-4ef4-bbd1-1c2c986cb504 jinja: 'Translate into English: {{translation["cs"]}} ||| {{translation["en"]}}' @@ -241,38 +257,65 @@ templates: name: !!python/tuple - translate-cs-en-target reference: '' - ac0ab68b-84e9-4b9d-9e50-c842a51b2f77: !Template + b9232185-faa3-4c04-857b-81d1b9866d3f: !Template answer_choices: null - id: ac0ab68b-84e9-4b9d-9e50-c842a51b2f77 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: b9232185-faa3-4c04-857b-81d1b9866d3f + jinja: 'English: {{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + ba9ef45a-e303-4b08-87fb-c7dd81ebdf98: !Template + answer_choices: null + id: ba9ef45a-e303-4b08-87fb-c7dd81ebdf98 + jinja: ' {{translation["en"]}} translates into Czech as: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-cs-target reference: '' - d3da03a4-7845-4c6f-86ec-a551749d03d8: !Template + bc27e6a2-b096-43a5-9443-5a285cd0f7b0: !Template answer_choices: null - id: d3da03a4-7845-4c6f-86ec-a551749d03d8 - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: bc27e6a2-b096-43a5-9443-5a285cd0f7b0 + jinja: 'Translate into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - translate-en-cs-target reference: '' - d3f2c452-c838-4abf-9142-fbebfefed494: !Template + d4c295d1-7f94-4e8d-a841-437f1fb3a8e5: !Template answer_choices: null - id: d3f2c452-c838-4abf-9142-fbebfefed494 + id: d4c295d1-7f94-4e8d-a841-437f1fb3a8e5 + jinja: '{{translation["cs"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + da65d991-c483-4ce2-aa07-93b2588c4512: !Template + answer_choices: null + id: da65d991-c483-4ce2-aa07-93b2588c4512 jinja: 'How do you say {{translation["cs"]}} in English? ||| {{translation["en"]}}' @@ -284,25 +327,38 @@ templates: name: !!python/tuple - how_to_say-cs-en-target reference: '' - d9386bcf-9dc9-448b-9f95-e01be14f1795: !Template + f4aa9206-271a-4e3b-94f7-b305b53eaabd: !Template answer_choices: null - id: d9386bcf-9dc9-448b-9f95-e01be14f1795 - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: f4aa9206-271a-4e3b-94f7-b305b53eaabd + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - translate_as_en-cs-source+target reference: '' - ebe24ff7-03b0-4a37-9da3-c3c715949cd7: !Template + f6143402-e4d4-494b-826d-b2733a9f1690: !Template answer_choices: null - id: ebe24ff7-03b0-4a37-9da3-c3c715949cd7 - jinja: '{{translation["cs"]}} translates into English as: + id: f6143402-e4d4-494b-826d-b2733a9f1690 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: GPT-3 paper + f6dcc950-8166-4806-b277-1d251c11a2fb: !Template + answer_choices: null + id: f6dcc950-8166-4806-b277-1d251c11a2fb + jinja: 'Czech: {{translation["cs"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target - reference: '' + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/de-en/templates.yaml b/promptsource/templates/wmt16/de-en/templates.yaml index 1bbf5f425..ff840bc88 100644 --- a/promptsource/templates/wmt16/de-en/templates.yaml +++ b/promptsource/templates/wmt16/de-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt16 subset: de-en templates: - 0cac8bad-ed93-405f-a810-cc4252b0148c: !Template + 011eb653-452c-4e29-bc28-41ead1f410b3: !Template answer_choices: null - id: 0cac8bad-ed93-405f-a810-cc4252b0148c + id: 011eb653-452c-4e29-bc28-41ead1f410b3 jinja: 'Translate into German: {{translation["en"]}} ||| {{translation["de"]}}' @@ -15,54 +15,54 @@ templates: name: !!python/tuple - translate-en-de-target reference: '' - 1348d039-2476-48ac-a2d2-7f66df53868d: !Template + 1193db70-93fd-4c9e-aa73-74d2b89d376b: !Template answer_choices: null - id: 1348d039-2476-48ac-a2d2-7f66df53868d - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 1193db70-93fd-4c9e-aa73-74d2b89d376b + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target - reference: '' - 1606b0f9-35b7-40a9-a46c-f416de1f4c82: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 131d4804-382c-4610-aae0-69ddd20bfa31: !Template answer_choices: null - id: 1606b0f9-35b7-40a9-a46c-f416de1f4c82 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 131d4804-382c-4610-aae0-69ddd20bfa31 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - version-en-de-source+target reference: '' - 1afad944-388b-4246-b83f-e10a3b3090b6: !Template + 16c3235a-aafd-4800-bc4e-010e9244af53: !Template answer_choices: null - id: 1afad944-388b-4246-b83f-e10a3b3090b6 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 16c3235a-aafd-4800-bc4e-010e9244af53 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target + - a_good_translation-de-en-target reference: '' - 229197cc-6240-4edb-b668-5efa298db15c: !Template + 1d30ca18-503b-4c06-8f1a-ec53213d4d96: !Template answer_choices: null - id: 229197cc-6240-4edb-b668-5efa298db15c - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 1d30ca18-503b-4c06-8f1a-ec53213d4d96 + jinja: 'How do you say {{translation["de"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -71,12 +71,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - how_to_say-de-en-target reference: '' - 380d03fd-7b06-435f-a00d-e640f95ac357: !Template + 1edbc64c-ee40-4c53-aa7a-78593b2b9366: !Template answer_choices: null - id: 380d03fd-7b06-435f-a00d-e640f95ac357 - jinja: 'How do you say {{translation["de"]}} in English? + id: 1edbc64c-ee40-4c53-aa7a-78593b2b9366 + jinja: 'German: {{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -85,24 +85,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 333f3945-9187-4e8c-9eb9-c26b6691089e: !Template + answer_choices: null + id: 333f3945-9187-4e8c-9eb9-c26b6691089e + jinja: 'How do you say {{translation["en"]}} in German? + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-de-target reference: '' - 39d60c43-fd32-4078-9fdc-1fb6e09b7ce6: !Template + 5b00be40-5344-43d6-b988-db3fefca8adf: !Template answer_choices: null - id: 39d60c43-fd32-4078-9fdc-1fb6e09b7ce6 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 5b00be40-5344-43d6-b988-db3fefca8adf + jinja: 'English: {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate_as_en-de-source+target reference: '' - 63e8d2c7-578b-444d-a4dd-a1037ed9a748: !Template + 5dad0b28-42ae-4adf-8ca6-9065da30840b: !Template answer_choices: null - id: 63e8d2c7-578b-444d-a4dd-a1037ed9a748 + id: 5dad0b28-42ae-4adf-8ca6-9065da30840b jinja: 'Given the following source text in German: {{translation["de"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -113,38 +128,37 @@ templates: name: !!python/tuple - a_good_translation-de-en-source+target reference: '' - 66367e82-d0a1-45e9-a04f-aa3937a5b5eb: !Template + 6476968d-e278-42ff-9f0a-48d705c58b88: !Template answer_choices: null - id: 66367e82-d0a1-45e9-a04f-aa3937a5b5eb - jinja: 'Translate English into German: {{translation["en"]}} - - ||| {{translation["de"]}}' + id: 6476968d-e278-42ff-9f0a-48d705c58b88 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget + - a_good_translation-en-de-target reference: '' - 66bd6d41-82fd-43ea-98c1-2e9e9ecdd3fd: !Template + 7ae0f06d-66a2-4e43-b993-0e2933b0acfc: !Template answer_choices: null - id: 66bd6d41-82fd-43ea-98c1-2e9e9ecdd3fd - jinja: 'If the German version says: {{translation["de"]}}; then the English version + id: 7ae0f06d-66a2-4e43-b993-0e2933b0acfc + jinja: 'If the original version says: {{translation["en"]}}; then the German version should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - version-en-de-target reference: '' - 702bbd13-29f7-4e83-95f5-09471856007b: !Template + 8aa85abc-974d-47ee-9b99-e534b100a77e: !Template answer_choices: null - id: 702bbd13-29f7-4e83-95f5-09471856007b + id: 8aa85abc-974d-47ee-9b99-e534b100a77e jinja: '{{translation["de"]}} translates into English as: ||| {{translation["en"]}}' @@ -156,37 +170,66 @@ templates: name: !!python/tuple - translate_as_de-en-target reference: '' - 738f96e7-1b92-42d0-b88c-99272e304e57: !Template + 8aec39a8-d6c1-42c9-b27e-c03c1bea40f7: !Template answer_choices: null - id: 738f96e7-1b92-42d0-b88c-99272e304e57 - jinja: 'English: {{translation["en"]}} translates into German as: + id: 8aec39a8-d6c1-42c9-b27e-c03c1bea40f7 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - version-de-en-target reference: '' - 8bfca2fd-9bbd-42a0-b924-8a1fc891b152: !Template + 955fd7a7-7e76-4983-9f89-864b31955076: !Template answer_choices: null - id: 8bfca2fd-9bbd-42a0-b924-8a1fc891b152 - jinja: 'Translate German into English: {{translation["de"]}} + id: 955fd7a7-7e76-4983-9f89-864b31955076 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - a52d0722-de2c-4cbc-89b1-430f9adf8b1d: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 95a4ed85-ee64-42fe-b9fa-15d204d1f240: !Template + answer_choices: null + id: 95a4ed85-ee64-42fe-b9fa-15d204d1f240 + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: GPT-3 paper + 9d5035f4-84f5-437e-9753-c5a3f2f6ce85: !Template answer_choices: null - id: a52d0722-de2c-4cbc-89b1-430f9adf8b1d + id: 9d5035f4-84f5-437e-9753-c5a3f2f6ce85 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-de-target + reference: '' + b2a5be40-6fee-4fd8-b585-991e79464f5c: !Template + answer_choices: null + id: b2a5be40-6fee-4fd8-b585-991e79464f5c jinja: 'Translate into English: {{translation["de"]}} ||| {{translation["en"]}}' @@ -198,118 +241,131 @@ templates: name: !!python/tuple - translate-de-en-target reference: '' - a7e7d7a3-791a-45de-91ac-62f4923296e9: !Template + cbb1cd17-b662-407d-b636-bf37995656e1: !Template answer_choices: null - id: a7e7d7a3-791a-45de-91ac-62f4923296e9 - jinja: 'If the original version says: {{translation["en"]}}; then the German version + id: cbb1cd17-b662-407d-b636-bf37995656e1 + jinja: 'If the German version says: {{translation["de"]}}; then the English version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - version-de-en-source+target reference: '' - afff8a67-9085-401a-8de7-992a8cd630ad: !Template + cec6977f-bd0a-4e11-8ca4-ab8858a69502: !Template answer_choices: null - id: afff8a67-9085-401a-8de7-992a8cd630ad - jinja: 'What is the English translation of : {{translation["de"]}} + id: cec6977f-bd0a-4e11-8ca4-ab8858a69502 + jinja: 'German: {{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target + - translate_as_de-en-source+target reference: '' - bb0bad5a-f598-48a4-871a-914da862f9fa: !Template + d28e05c8-3bff-4579-a173-ae39b1e4eec1: !Template answer_choices: null - id: bb0bad5a-f598-48a4-871a-914da862f9fa - jinja: 'How do you say {{translation["en"]}} in German? + id: d28e05c8-3bff-4579-a173-ae39b1e4eec1 + jinja: '{{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - cc00a59b-9da3-4167-8a2d-c84670899941: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d8546622-b890-467e-9a2a-1fbefb9239a6: !Template answer_choices: null - id: cc00a59b-9da3-4167-8a2d-c84670899941 - jinja: ' {{translation["en"]}} translates into German as: + id: d8546622-b890-467e-9a2a-1fbefb9239a6 + jinja: 'Translate English into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - translate-en-de-source+starget reference: '' - d0b83143-620c-4b18-9a27-c2dd9b82045b: !Template + e13131f0-bc41-4e29-a552-e2f3e99abffe: !Template answer_choices: null - id: d0b83143-620c-4b18-9a27-c2dd9b82045b - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: e13131f0-bc41-4e29-a552-e2f3e99abffe + jinja: 'Translate German into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target - reference: '' - d9857ba6-8dea-40f0-ad61-757d21d08e05: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e397bc2f-425e-4afe-a543-c8973054ef52: !Template answer_choices: null - id: d9857ba6-8dea-40f0-ad61-757d21d08e05 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: e397bc2f-425e-4afe-a543-c8973054ef52 + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - eb6e2d75-18ea-400b-baa5-650f7e3b6a37: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + e7d42462-80ec-498e-916a-56ffb65dfb8c: !Template answer_choices: null - id: eb6e2d75-18ea-400b-baa5-650f7e3b6a37 - jinja: 'German: {{translation["de"]}} translates into English as: + id: e7d42462-80ec-498e-916a-56ffb65dfb8c + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - a_good_translation-en-de-source+target reference: '' - f2f1d8c9-e5a0-4443-9343-c4e470346cba: !Template + f231daf4-2277-4a91-8be5-3319fbeb621b: !Template answer_choices: null - id: f2f1d8c9-e5a0-4443-9343-c4e470346cba - jinja: 'What is the German translation of: {{translation["en"]}} + id: f231daf4-2277-4a91-8be5-3319fbeb621b + jinja: 'What is the English translation of : {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: '' + - gpt-3-de-en-target + reference: GPT-3 paper + ffc5597a-dea3-4530-982a-81b2f4f8ec74: !Template + answer_choices: null + id: ffc5597a-dea3-4530-982a-81b2f4f8ec74 + jinja: '{{translation["en"]}} = German: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/fi-en/templates.yaml b/promptsource/templates/wmt16/fi-en/templates.yaml index 4df5027f2..32d675257 100644 --- a/promptsource/templates/wmt16/fi-en/templates.yaml +++ b/promptsource/templates/wmt16/fi-en/templates.yaml @@ -1,37 +1,52 @@ dataset: wmt16 subset: fi-en templates: - 009a2c37-87d1-4384-9301-a53b4ab811de: !Template + 149671ed-74c4-46eb-a89c-fd1868585d84: !Template answer_choices: null - id: 009a2c37-87d1-4384-9301-a53b4ab811de - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 149671ed-74c4-46eb-a89c-fd1868585d84 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-src+starget - reference: '' - 033f1d74-9bee-4851-8fd7-b087a7823c4e: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 18d30167-1063-42e5-9ec7-5d640e1f1b12: !Template answer_choices: null - id: 033f1d74-9bee-4851-8fd7-b087a7823c4e - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 18d30167-1063-42e5-9ec7-5d640e1f1b12 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - version-en-fi-source+target reference: '' - 13525c5d-3c9b-4554-a3bf-88acecb925d4: !Template + 1a1dd83a-cbf4-4275-af5e-ea820a00e7e0: !Template + answer_choices: null + id: 1a1dd83a-cbf4-4275-af5e-ea820a00e7e0 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 1a6db208-8811-485d-bd07-2e8c137e3b42: !Template answer_choices: null - id: 13525c5d-3c9b-4554-a3bf-88acecb925d4 + id: 1a6db208-8811-485d-bd07-2e8c137e3b42 jinja: 'Translate into English: {{translation["fi"]}} ||| {{translation["en"]}}' @@ -43,25 +58,39 @@ templates: name: !!python/tuple - translate-fi-en-target reference: '' - 1f8fd96b-3684-44a3-8bef-6e2ffc117347: !Template + 34fb63e0-3923-415a-86f2-159a8278a98d: !Template answer_choices: null - id: 1f8fd96b-3684-44a3-8bef-6e2ffc117347 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: 34fb63e0-3923-415a-86f2-159a8278a98d + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target + - gpt-3-en-fi-source+target + reference: GPT 3 paper + 39a2e39b-22a2-45b5-b919-88ec767ee423: !Template + answer_choices: null + id: 39a2e39b-22a2-45b5-b919-88ec767ee423 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target reference: '' - 368b581c-c592-4bfd-a79e-2572d2c010be: !Template + 3d238ee8-502f-42c7-a20c-714caa6a95dc: !Template answer_choices: null - id: 368b581c-c592-4bfd-a79e-2572d2c010be - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: 3d238ee8-502f-42c7-a20c-714caa6a95dc + jinja: ' {{translation["en"]}} translates into Finnish as: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -70,25 +99,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target + - translate_as_en-fi-target reference: '' - 3bac8137-9ea7-4eb7-93af-44f61eedb495: !Template + 4192a66e-91e7-4c1a-ba77-6e21716848e6: !Template answer_choices: null - id: 3bac8137-9ea7-4eb7-93af-44f61eedb495 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: 4192a66e-91e7-4c1a-ba77-6e21716848e6 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-src+target - reference: '' - 3dae3855-5014-4d1b-834c-713224698305: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5716cdc9-23bc-4d73-9652-e09868f6f21a: !Template answer_choices: null - id: 3dae3855-5014-4d1b-834c-713224698305 + id: 5716cdc9-23bc-4d73-9652-e09868f6f21a jinja: 'English: {{translation["en"]}} translates into Finnish as: ||| {{translation["fi"]}}' @@ -100,23 +129,23 @@ templates: name: !!python/tuple - translate_as_en-fi-source+target reference: '' - 5f36e461-251e-4690-9d23-3500380b965a: !Template + 5c474ea2-c34c-4049-9a4e-97ef025e4085: !Template answer_choices: null - id: 5f36e461-251e-4690-9d23-3500380b965a - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 5c474ea2-c34c-4049-9a4e-97ef025e4085 + jinja: 'Translate into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target + - translate-en-fi-target reference: '' - 6f1bb4e0-5391-4357-87ca-d906594587ef: !Template + 6a226e13-cd90-42aa-b7b2-e98d0c542513: !Template answer_choices: null - id: 6f1bb4e0-5391-4357-87ca-d906594587ef + id: 6a226e13-cd90-42aa-b7b2-e98d0c542513 jinja: 'What is the Finnish translation of: {{translation["en"]}} ||| {{translation["fi"]}}' @@ -127,26 +156,26 @@ templates: original_task: true name: !!python/tuple - gpt-3-en-fi-target - reference: '' - 83cadcf9-dc67-425b-9cb4-07c4bfc230d2: !Template + reference: GPT-3 paper + 6de62fd5-737f-4c86-bb36-fe9d741dc992: !Template answer_choices: null - id: 83cadcf9-dc67-425b-9cb4-07c4bfc230d2 - jinja: 'If the original version says: {{translation["fi"]}}; then the English + id: 6de62fd5-737f-4c86-bb36-fe9d741dc992 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - version-en-fi-target reference: '' - 8cb88018-e934-4717-aeb1-154314b7d640: !Template + 8275e339-4924-4525-a6a3-c8855d02ec7a: !Template answer_choices: null - id: 8cb88018-e934-4717-aeb1-154314b7d640 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: 8275e339-4924-4525-a6a3-c8855d02ec7a + jinja: 'Translate English into Finnish: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -155,13 +184,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - translate-en-fi-source+starget reference: '' - 8f9765ec-86cf-49f2-8ba3-3329465977da: !Template + 8eaa9792-9b2e-4350-a362-9da0aca71110: !Template answer_choices: null - id: 8f9765ec-86cf-49f2-8ba3-3329465977da - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 8eaa9792-9b2e-4350-a362-9da0aca71110 + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-target + reference: GPT-3 paper + 904df1cb-a22f-4dc3-a4a1-65a0255f6a36: !Template + answer_choices: null + id: 904df1cb-a22f-4dc3-a4a1-65a0255f6a36 + jinja: 'English: {{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 94309855-afd5-4235-8731-b8c6959d0c27: !Template + answer_choices: null + id: 94309855-afd5-4235-8731-b8c6959d0c27 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -170,12 +226,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - translate_as_fi-en-source+target reference: '' - 9b008796-65d0-4efc-83b8-69bed01e9ebc: !Template + 9a096544-5a59-4f74-8b0f-760aab4123b1: !Template answer_choices: null - id: 9b008796-65d0-4efc-83b8-69bed01e9ebc - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 9a096544-5a59-4f74-8b0f-760aab4123b1 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -184,26 +241,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - a_good_translation-en-fi-source+target reference: '' - 9bf2ce4f-5b6a-4778-8708-1e3f9b86cdff: !Template + a09933b1-c90d-4c4b-beb8-f8bb3492d70e: !Template answer_choices: null - id: 9bf2ce4f-5b6a-4778-8708-1e3f9b86cdff - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: a09933b1-c90d-4c4b-beb8-f8bb3492d70e + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - how_to_say-fi-en-target reference: '' - a5dbdded-9234-4ebd-85d5-93ee37d382c8: !Template + b10cd627-6233-4035-b4e2-b0b94bf84334: !Template answer_choices: null - id: a5dbdded-9234-4ebd-85d5-93ee37d382c8 + id: b10cd627-6233-4035-b4e2-b0b94bf84334 jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -214,23 +270,23 @@ templates: name: !!python/tuple - a_good_translation-en-fi-target reference: '' - ac4a34bc-d850-4ae1-8275-fc2af8eb0324: !Template + b16016c7-2050-477a-aa2d-2866d203d045: !Template answer_choices: null - id: ac4a34bc-d850-4ae1-8275-fc2af8eb0324 - jinja: '{{translation["fi"]}} translates into English as: + id: b16016c7-2050-477a-aa2d-2866d203d045 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target + - how_to_say-en-fi-target reference: '' - ac6ad43d-d441-442b-b62c-ae11e1c197c8: !Template + b250a904-f24b-41a2-b38e-93eca25bd103: !Template answer_choices: null - id: ac6ad43d-d441-442b-b62c-ae11e1c197c8 + id: b250a904-f24b-41a2-b38e-93eca25bd103 jinja: 'Translate Finnish into English: {{translation["fi"]}} ||| {{translation["en"]}}' @@ -242,9 +298,9 @@ templates: name: !!python/tuple - translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - b92e677c-e5a0-475a-945a-41747d33ce7f: !Template + b28ba135-022b-4ab9-9c99-79e1e286fe4a: !Template answer_choices: null - id: b92e677c-e5a0-475a-945a-41747d33ce7f + id: b28ba135-022b-4ab9-9c99-79e1e286fe4a jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -255,54 +311,54 @@ templates: name: !!python/tuple - a_good_translation-fi-en-source+target reference: '' - c06b082e-1028-4ec8-8ce1-6e6bd5ff9331: !Template + b435a4aa-d75d-4ecb-a8ee-a6c9ae589171: !Template answer_choices: null - id: c06b082e-1028-4ec8-8ce1-6e6bd5ff9331 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + id: b435a4aa-d75d-4ecb-a8ee-a6c9ae589171 + jinja: 'If the original version says: {{translation["fi"]}}; then the English version should say: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - version-fi-en-target reference: '' - c714391d-7e6a-4621-8829-c4d08a70c64f: !Template + bd99a7cf-df50-4355-adeb-3573fd935797: !Template answer_choices: null - id: c714391d-7e6a-4621-8829-c4d08a70c64f - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: bd99a7cf-df50-4355-adeb-3573fd935797 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target - reference: '' - cdc5df34-d605-4eaf-9baa-cdc11952654f: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d03d3586-e7a9-4ff8-b779-fc1504260505: !Template answer_choices: null - id: cdc5df34-d605-4eaf-9baa-cdc11952654f - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: d03d3586-e7a9-4ff8-b779-fc1504260505 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-src+target + - a_good_translation-fi-en-target reference: '' - ff2ea6cd-1bb6-4192-8b29-8d9ab5fdb3fe: !Template + e7edf827-72d4-4e6e-8947-1cf28932715b: !Template answer_choices: null - id: ff2ea6cd-1bb6-4192-8b29-8d9ab5fdb3fe - jinja: 'How do you say {{translation["fi"]}} in English? + id: e7edf827-72d4-4e6e-8947-1cf28932715b + jinja: '{{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - translate_as_fi-en-target reference: '' diff --git a/promptsource/templates/wmt16/ro-en/templates.yaml b/promptsource/templates/wmt16/ro-en/templates.yaml index be38d7df7..acfbc141b 100644 --- a/promptsource/templates/wmt16/ro-en/templates.yaml +++ b/promptsource/templates/wmt16/ro-en/templates.yaml @@ -1,123 +1,151 @@ dataset: wmt16 subset: ro-en templates: - 2d942085-1c5a-4b3d-956a-c50385ccb2dc: !Template + 05b08ca5-11eb-4eed-a634-d8e146cbffbd: !Template answer_choices: null - id: 2d942085-1c5a-4b3d-956a-c50385ccb2dc - jinja: 'How do you say {{translation["en"]}} in Romanian? + id: 05b08ca5-11eb-4eed-a634-d8e146cbffbd + jinja: 'Given the following passage: {{translation["ro"]}} , a good English translation + is: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ro-target + - a_good_translation-ro-en-target reference: '' - 2de0143d-57c2-4379-8b87-f11cfb386039: !Template + 0ae186cb-de3f-47dc-b1a6-d787844a79fd: !Template answer_choices: null - id: 2de0143d-57c2-4379-8b87-f11cfb386039 - jinja: 'What is the Romanian translation of: {{translation["en"]}} + id: 0ae186cb-de3f-47dc-b1a6-d787844a79fd + jinja: 'Given the following passage: {{translation["en"]}} , a good Romanian translation + is: ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ro-target + reference: '' + 0e7422d6-244f-40d9-be09-6d9248affa6c: !Template + answer_choices: null + id: 0e7422d6-244f-40d9-be09-6d9248affa6c + jinja: 'Translate Romanian into English: {{translation["ro"]}} - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ro-target + - translate-ro-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 3743872c-f834-4c4d-a32c-1ad035f8b7cd: !Template + answer_choices: null + id: 3743872c-f834-4c4d-a32c-1ad035f8b7cd + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Romanian translation is: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ro-source+target reference: '' - 2edfd6ed-0233-4062-aa9c-66bef54b2a34: !Template + 3ed2a5f2-c6ea-4b18-a45e-045eafe0592e: !Template answer_choices: null - id: 2edfd6ed-0233-4062-aa9c-66bef54b2a34 - jinja: 'If the original version says: {{translation["ro"]}}; then the English - version should say: + id: 3ed2a5f2-c6ea-4b18-a45e-045eafe0592e + jinja: ' {{translation["en"]}} translates into Romanian as: - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ro-en-target + - translate_as_en-ro-target reference: '' - 48998b22-4bad-4a98-8d0d-ad90584845f2: !Template + 42b4bf2d-1bb5-4669-b041-dc761dfb84eb: !Template answer_choices: null - id: 48998b22-4bad-4a98-8d0d-ad90584845f2 - jinja: 'Translate Romanian into English: {{translation["ro"]}} + id: 42b4bf2d-1bb5-4669-b041-dc761dfb84eb + jinja: 'Romanian: {{translation["ro"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ro-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 4cd03022-c0e5-4d80-810d-f99d345d0ce4: !Template + - xglm-ro-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 465d16b0-1dd0-48d4-a07b-f99b262feeb5: !Template answer_choices: null - id: 4cd03022-c0e5-4d80-810d-f99d345d0ce4 - jinja: 'Translate English into Romanian: {{translation["en"]}} + id: 465d16b0-1dd0-48d4-a07b-f99b262feeb5 + jinja: '{{translation["en"]}} = Romanian: - ||| {{translation["ro"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ro-src+starget - reference: '' - 4d0f8d04-3149-4696-9764-d194afa31936: !Template + - xglm-en-ro-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4e2fcab2-05fb-4099-a038-9c585b9b4be7: !Template answer_choices: null - id: 4d0f8d04-3149-4696-9764-d194afa31936 - jinja: 'Translate into English: {{translation["ro"]}} + id: 4e2fcab2-05fb-4099-a038-9c585b9b4be7 + jinja: 'English: {{translation["en"]}} translates into Romanian as: - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ro-en-target + - translate_as_en-ro-source+target reference: '' - 56382179-2a48-446d-8de3-4701a888d122: !Template + 5bf16978-6b7d-468a-b84f-943f0179a548: !Template answer_choices: null - id: 56382179-2a48-446d-8de3-4701a888d122 - jinja: '{{translation["ro"]}} translates into English as: + id: 5bf16978-6b7d-468a-b84f-943f0179a548 + jinja: 'English: {{translation["en"]}} = Romanian: - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ro-en-target - reference: '' - 6179fada-a56f-4a46-aec7-4b637a77f24f: !Template + - xglm-en-ro-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 62e17c10-5d28-4822-88f2-5eaec1d09c9e: !Template answer_choices: null - id: 6179fada-a56f-4a46-aec7-4b637a77f24f - jinja: 'What is the English translation of : {{translation["ro"]}} + id: 62e17c10-5d28-4822-88f2-5eaec1d09c9e + jinja: 'What is the Romanian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ro-en-target - reference: '' - 68bb72ef-c866-467f-abe9-d6c89f58175a: !Template + - gpt-3-en-ro-source+target + reference: GPT 3 paper + 6eade7af-18af-40b4-8369-7ca24f043914: !Template answer_choices: null - id: 68bb72ef-c866-467f-abe9-d6c89f58175a - jinja: 'If the original version says: {{translation["en"]}}; then the Romanian + id: 6eade7af-18af-40b4-8369-7ca24f043914 + jinja: 'If the English version says: {{translation["en"]}}; then the Romanian version should say: ||| {{translation["ro"]}}' @@ -127,27 +155,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-ro-target + - version-en-ro-source+target reference: '' - 6ed88276-594a-4c97-a8c0-b637e154d5b1: !Template + 703343c4-14bb-44b2-aed8-034433d1d251: !Template answer_choices: null - id: 6ed88276-594a-4c97-a8c0-b637e154d5b1 - jinja: 'Given the following passage: {{translation["ro"]}} , a good English translation - is: + id: 703343c4-14bb-44b2-aed8-034433d1d251 + jinja: 'How do you say {{translation["en"]}} in Romanian? - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ro-en-target + - how_to_say-en-ro-target reference: '' - 7bffa288-bdd0-4b0f-8738-44540b1154e7: !Template + 8789e689-e914-4a09-9614-d27b38f05528: !Template answer_choices: null - id: 7bffa288-bdd0-4b0f-8738-44540b1154e7 - jinja: 'What is the Romanian translation of the English sentence: {{translation["en"]}} + id: 8789e689-e914-4a09-9614-d27b38f05528 + jinja: 'Translate English into Romanian: {{translation["en"]}} ||| {{translation["ro"]}}' metadata: !TemplateMetadata @@ -156,108 +183,125 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ro-src+target + - translate-en-ro-source+starget reference: '' - 8142e62d-c4a6-4cf9-9d66-59ba23aa177e: !Template + 981b526d-aad9-497b-8563-c523b640c12e: !Template answer_choices: null - id: 8142e62d-c4a6-4cf9-9d66-59ba23aa177e - jinja: 'How do you say {{translation["ro"]}} in English? + id: 981b526d-aad9-497b-8563-c523b640c12e + jinja: 'If the Romanian version says: {{translation["ro"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ro-en-target + - version-ro-en-source+target reference: '' - 8871ac3e-f8a1-491f-89cd-366d6133cb76: !Template + 9c5af958-102b-4fc7-a477-b7fad827f07f: !Template answer_choices: null - id: 8871ac3e-f8a1-491f-89cd-366d6133cb76 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Romanian translation is: + id: 9c5af958-102b-4fc7-a477-b7fad827f07f + jinja: 'Translate into English: {{translation["ro"]}} - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ro-source+target + - translate-ro-en-target reference: '' - 899f7360-9314-4057-a6ba-35b1466d22c1: !Template + a40eb7aa-fe7e-4fd6-bc2f-dcd295a2e96d: !Template answer_choices: null - id: 899f7360-9314-4057-a6ba-35b1466d22c1 - jinja: ' {{translation["en"]}} translates into Romanian as: + id: a40eb7aa-fe7e-4fd6-bc2f-dcd295a2e96d + jinja: '{{translation["ro"]}} = English: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ro-target + - xglm-ro-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a4c38371-d526-4783-8213-39df0ec8ff12: !Template + answer_choices: null + id: a4c38371-d526-4783-8213-39df0ec8ff12 + jinja: 'If the original version says: {{translation["en"]}}; then the Romanian + version should say: + + ||| {{translation["ro"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-ro-target reference: '' - a0726d07-8f0a-4b1e-882f-b10368bfa0f0: !Template + b01b7756-3879-4814-9909-0cd565d172f5: !Template answer_choices: null - id: a0726d07-8f0a-4b1e-882f-b10368bfa0f0 - jinja: 'Given the following passage: {{translation["en"]}} , a good Romanian translation - is: ||| {{translation["ro"]}}' + id: b01b7756-3879-4814-9909-0cd565d172f5 + jinja: 'Given the following source text in Romanian: {{translation["ro"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ro-target + - a_good_translation-ro-en-source+target reference: '' - b5c54c2c-1232-4779-bb58-919f81d8cfd6: !Template + b64b25c0-af3e-4247-bc53-8c40048cd9e7: !Template answer_choices: null - id: b5c54c2c-1232-4779-bb58-919f81d8cfd6 - jinja: 'Translate into Romanian: {{translation["en"]}} + id: b64b25c0-af3e-4247-bc53-8c40048cd9e7 + jinja: 'How do you say {{translation["ro"]}} in English? - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ro-target + - how_to_say-ro-en-target reference: '' - b92197f6-88bf-4375-8d43-48a784d4fd98: !Template + b6920ff2-7442-4d06-ad2a-3dd6d58a53ee: !Template answer_choices: null - id: b92197f6-88bf-4375-8d43-48a784d4fd98 - jinja: 'Given the following source text in Romanian: {{translation["ro"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: b6920ff2-7442-4d06-ad2a-3dd6d58a53ee + jinja: 'If the original version says: {{translation["ro"]}}; then the English + version should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ro-en-source+target + - version-ro-en-target reference: '' - da7917f0-aad3-4a6b-83b1-6c94c7a5c3e9: !Template + bcea9f08-def5-4711-8a91-7f5f2191e4cb: !Template answer_choices: null - id: da7917f0-aad3-4a6b-83b1-6c94c7a5c3e9 - jinja: 'English: {{translation["en"]}} translates into Romanian as: + id: bcea9f08-def5-4711-8a91-7f5f2191e4cb + jinja: '{{translation["ro"]}} translates into English as: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ro-source+target + - translate_as_ro-en-target reference: '' - dc349c72-3b44-4fa6-8998-099c86e07ecd: !Template + d3eb0c1d-242a-401e-8e0c-97ad9ef6bf06: !Template answer_choices: null - id: dc349c72-3b44-4fa6-8998-099c86e07ecd + id: d3eb0c1d-242a-401e-8e0c-97ad9ef6bf06 jinja: 'Romanian: {{translation["ro"]}} translates into English as: ||| {{translation["en"]}}' @@ -269,47 +313,59 @@ templates: name: !!python/tuple - translate_as_ro-en-source+target reference: '' - dc531152-e769-4b15-8c08-07829e7c7da7: !Template + dbd449cd-bc4f-44e0-9634-a69ecf5a5c1b: !Template answer_choices: null - id: dc531152-e769-4b15-8c08-07829e7c7da7 - jinja: 'If the English version says: {{translation["en"]}}; then the Romanian - version should say: + id: dbd449cd-bc4f-44e0-9634-a69ecf5a5c1b + jinja: 'What is the English translation of : {{translation["ro"]}} - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ro-source+target - reference: '' - e61371c7-6627-4578-bcae-9127a49ea772: !Template + - gpt-3-ro-en-target + reference: GPT-3 paper + f4d2dea9-6b81-4709-a2da-fcd21bde5107: !Template answer_choices: null - id: e61371c7-6627-4578-bcae-9127a49ea772 - jinja: 'What is the English translation of the Romanian sentence: {{translation["ro"]}} + id: f4d2dea9-6b81-4709-a2da-fcd21bde5107 + jinja: 'Translate into Romanian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ro-en-src+target + - translate-en-ro-target reference: '' - f38ceb60-50e2-4b78-9c80-2bea7a5231c0: !Template + fc553e43-27d2-4ed5-8c95-743972d91a47: !Template answer_choices: null - id: f38ceb60-50e2-4b78-9c80-2bea7a5231c0 - jinja: 'If the Romanian version says: {{translation["ro"]}}; then the English - version should say: + id: fc553e43-27d2-4ed5-8c95-743972d91a47 + jinja: 'What is the Romanian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ro-en-source+target - reference: '' + - gpt-3-en-ro-target + reference: GPT-3 paper + fc9a94ab-3355-4af0-9960-387303ab513b: !Template + answer_choices: null + id: fc9a94ab-3355-4af0-9960-387303ab513b + jinja: 'What is the English translation of the Romanian sentence: {{translation["ro"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-ro-en-source+target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt16/ru-en/templates.yaml b/promptsource/templates/wmt16/ru-en/templates.yaml index 948310c91..78418f0b5 100644 --- a/promptsource/templates/wmt16/ru-en/templates.yaml +++ b/promptsource/templates/wmt16/ru-en/templates.yaml @@ -1,25 +1,24 @@ dataset: wmt16 subset: ru-en templates: - 12af9256-262a-4b74-91be-ff32c1afbd3c: !Template + 03d02f14-f91d-421e-8fae-a91fc908dbab: !Template answer_choices: null - id: 12af9256-262a-4b74-91be-ff32c1afbd3c - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 03d02f14-f91d-421e-8fae-a91fc908dbab + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - translate_as_ru-en-target reference: '' - 1c5d3480-e1a4-44f1-9549-d5b3d5d8490e: !Template + 05d68683-b85b-43e4-9b61-7f293c736ea0: !Template answer_choices: null - id: 1c5d3480-e1a4-44f1-9549-d5b3d5d8490e - jinja: 'Translate into Russian: {{translation["en"]}} + id: 05d68683-b85b-43e4-9b61-7f293c736ea0 + jinja: 'What is the Russian translation of: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -28,11 +27,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target - reference: '' - 2069bf34-0c3f-4512-b87a-71a4e051e10e: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 1666915a-19e0-47a1-a935-52645db05501: !Template + answer_choices: null + id: 1666915a-19e0-47a1-a935-52645db05501 + jinja: 'Translate Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 19c0c34c-25a8-481a-8c7e-74d6032a1e26: !Template answer_choices: null - id: 2069bf34-0c3f-4512-b87a-71a4e051e10e + id: 19c0c34c-25a8-481a-8c7e-74d6032a1e26 jinja: 'If the original version says: {{translation["ru"]}}; then the English version should say: @@ -45,52 +58,53 @@ templates: name: !!python/tuple - version-ru-en-target reference: '' - 267de81e-0791-419f-b9be-f9461b70fc4b: !Template + 1c3f4328-4b9c-45a8-a62b-97839c9b5429: !Template answer_choices: null - id: 267de81e-0791-419f-b9be-f9461b70fc4b - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 1c3f4328-4b9c-45a8-a62b-97839c9b5429 + jinja: 'Translate English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - translate-en-ru-source+starget reference: '' - 2825d558-0af8-4259-9258-372254a5f7c6: !Template + 28128ce4-3426-4fdc-84c7-9861387933d3: !Template answer_choices: null - id: 2825d558-0af8-4259-9258-372254a5f7c6 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 28128ce4-3426-4fdc-84c7-9861387933d3 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target - reference: '' - 2c197bee-b468-4fa8-a2a9-07d023a86e36: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 2e10725a-66ab-41d7-af3b-23013e99d4f7: !Template answer_choices: null - id: 2c197bee-b468-4fa8-a2a9-07d023a86e36 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 2e10725a-66ab-41d7-af3b-23013e99d4f7 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget + - a_good_translation-ru-en-target reference: '' - 3e92539c-1589-4684-a0d8-fef5451727d0: !Template + 36d643f8-5e9f-4536-ad88-fb72c317b576: !Template answer_choices: null - id: 3e92539c-1589-4684-a0d8-fef5451727d0 - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 36d643f8-5e9f-4536-ad88-fb72c317b576 + jinja: 'Russian: {{translation["ru"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -99,53 +113,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target - reference: '' - 43e99736-f6f3-4b11-aec1-3629f7b4d924: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3cbae036-ede7-47dd-b3bb-eeecaaaf47c0: !Template answer_choices: null - id: 43e99736-f6f3-4b11-aec1-3629f7b4d924 - jinja: 'Translate into English: {{translation["ru"]}} + id: 3cbae036-ede7-47dd-b3bb-eeecaaaf47c0 + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target - reference: '' - 4e5de29d-81f1-4742-8c47-4a7185639afd: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 65300887-6919-4804-9594-c7e9880fb62c: !Template answer_choices: null - id: 4e5de29d-81f1-4742-8c47-4a7185639afd - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 65300887-6919-4804-9594-c7e9880fb62c + jinja: 'How do you say {{translation["ru"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target + - how_to_say-ru-en-target reference: '' - 4f3a5dd7-6180-4140-b70d-f455dc5b9908: !Template + 6fde4437-43c8-4e4e-9a64-b961036b8977: !Template answer_choices: null - id: 4f3a5dd7-6180-4140-b70d-f455dc5b9908 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 6fde4437-43c8-4e4e-9a64-b961036b8977 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - version-en-ru-target reference: '' - 4f9cc10f-b20f-4fbb-8677-717a6dd075c2: !Template + 76eea0de-6fd9-485f-9ebf-bc4a51e4d600: !Template answer_choices: null - id: 4f9cc10f-b20f-4fbb-8677-717a6dd075c2 + id: 76eea0de-6fd9-485f-9ebf-bc4a51e4d600 jinja: 'If the English version says: {{translation["en"]}}; then the Russian version should say: @@ -158,53 +173,66 @@ templates: name: !!python/tuple - version-en-ru-source+target reference: '' - 60858979-aa78-427a-9a7f-d60d92ae4dc2: !Template + 773390d4-3602-46e5-b76a-c09ac8e957ca: !Template answer_choices: null - id: 60858979-aa78-427a-9a7f-d60d92ae4dc2 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 773390d4-3602-46e5-b76a-c09ac8e957ca + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target + - a_good_translation-en-ru-source+target reference: '' - 635bee5f-d63b-465e-adf7-b21528af3ebe: !Template + 78c8a6cc-24b3-4c2c-a9ce-daf57b1c712c: !Template answer_choices: null - id: 635bee5f-d63b-465e-adf7-b21528af3ebe - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 78c8a6cc-24b3-4c2c-a9ce-daf57b1c712c + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target + - gpt-3-ru-en-target + reference: GPT-3 paper + 7cf6777c-e32c-4cd9-b794-fc86e29e8083: !Template + answer_choices: null + id: 7cf6777c-e32c-4cd9-b794-fc86e29e8083 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target reference: '' - 6828bba2-fb1f-4ffb-b775-2d8cde0e24a9: !Template + 90c0449c-8364-4663-bee9-8217770fcea0: !Template answer_choices: null - id: 6828bba2-fb1f-4ffb-b775-2d8cde0e24a9 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 90c0449c-8364-4663-bee9-8217770fcea0 + jinja: 'Translate into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - translate-en-ru-target reference: '' - 74ab5d76-1cdc-4864-b10f-6d40f9676717: !Template + a0d1d119-df3c-437a-8940-a84fa73b9eeb: !Template answer_choices: null - id: 74ab5d76-1cdc-4864-b10f-6d40f9676717 - jinja: '{{translation["ru"]}} translates into English as: + id: a0d1d119-df3c-437a-8940-a84fa73b9eeb + jinja: '{{translation["ru"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -213,25 +241,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a8afdc6f-1b33-4773-8bda-37a01030cef2: !Template + answer_choices: null + id: a8afdc6f-1b33-4773-8bda-37a01030cef2 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target reference: '' - 75dd6cf8-1b57-4976-9691-2e9e0ddc5c37: !Template + aa4db5da-1829-40c2-9ca4-634f0279b579: !Template answer_choices: null - id: 75dd6cf8-1b57-4976-9691-2e9e0ddc5c37 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: aa4db5da-1829-40c2-9ca4-634f0279b579 + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - translate_as_ru-en-source+target reference: '' - 8c18f538-d59a-4bcc-bab7-b655ba847372: !Template + b191cdab-98c6-44cf-a864-1fa8a73c0f78: !Template answer_choices: null - id: 8c18f538-d59a-4bcc-bab7-b655ba847372 + id: b191cdab-98c6-44cf-a864-1fa8a73c0f78 jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version should say: @@ -244,36 +285,51 @@ templates: name: !!python/tuple - version-ru-en-source+target reference: '' - cd6e3dbd-36fa-4774-9bac-78d8d2b50a0d: !Template + c94ef825-4ad3-4128-9cba-29581906ccdf: !Template answer_choices: null - id: cd6e3dbd-36fa-4774-9bac-78d8d2b50a0d - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: c94ef825-4ad3-4128-9cba-29581906ccdf + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - eb4dcd70-1bf3-4a96-abaf-38521fa957ee: !Template + - translate_as_en-ru-target + reference: '' + e46bf114-cad8-4e1f-a5bc-230fc7da194e: !Template answer_choices: null - id: eb4dcd70-1bf3-4a96-abaf-38521fa957ee - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: e46bf114-cad8-4e1f-a5bc-230fc7da194e + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target - reference: '' - edef1b19-1f34-4a6c-bc06-516a0f5baab4: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + ef0bdff5-c6d9-4fdd-8269-d67bbbebedd6: !Template + answer_choices: null + id: ef0bdff5-c6d9-4fdd-8269-d67bbbebedd6 + jinja: 'English: {{translation["en"]}} = Russian: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + f0c55f7b-9021-4811-8966-000e2654cd39: !Template answer_choices: null - id: edef1b19-1f34-4a6c-bc06-516a0f5baab4 + id: f0c55f7b-9021-4811-8966-000e2654cd39 jinja: 'How do you say {{translation["en"]}} in Russian? ||| {{translation["ru"]}}' @@ -285,10 +341,10 @@ templates: name: !!python/tuple - how_to_say-en-ru-target reference: '' - ee42c631-6217-4bf5-b0a4-0a1dfa2c4651: !Template + f42e0011-a338-47d6-971f-57cf1de83a82: !Template answer_choices: null - id: ee42c631-6217-4bf5-b0a4-0a1dfa2c4651 - jinja: ' {{translation["en"]}} translates into Russian as: + id: f42e0011-a338-47d6-971f-57cf1de83a82 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -297,19 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate_as_en-ru-source+target reference: '' - f31e5d99-55c7-4369-a07c-0efa4d5e4ef3: !Template + f9c2e846-0274-4933-bd9f-615e572e376d: !Template answer_choices: null - id: f31e5d99-55c7-4369-a07c-0efa4d5e4ef3 - jinja: 'How do you say {{translation["ru"]}} in English? + id: f9c2e846-0274-4933-bd9f-615e572e376d + jinja: 'Translate into English: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate-ru-en-target reference: '' diff --git a/promptsource/templates/wmt16/tr-en/templates.yaml b/promptsource/templates/wmt16/tr-en/templates.yaml index ca63b34b4..4a8b2b6ae 100644 --- a/promptsource/templates/wmt16/tr-en/templates.yaml +++ b/promptsource/templates/wmt16/tr-en/templates.yaml @@ -1,67 +1,67 @@ dataset: wmt16 subset: tr-en templates: - 0930ab6b-c545-45cb-afd2-7776011bf61b: !Template + 03be6519-f6d4-4fb9-81bf-a6c32e54f624: !Template answer_choices: null - id: 0930ab6b-c545-45cb-afd2-7776011bf61b - jinja: 'English: {{translation["en"]}} translates into Turkish as: + id: 03be6519-f6d4-4fb9-81bf-a6c32e54f624 + jinja: 'What is the English translation of : {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target - reference: '' - 0f4540f4-8b32-4ff0-b731-e8ac6306525b: !Template + - gpt-3-tr-en-target + reference: GPT-3 paper + 07cfea8a-6d20-4719-a4ba-2ef8af3c416f: !Template answer_choices: null - id: 0f4540f4-8b32-4ff0-b731-e8ac6306525b - jinja: 'How do you say {{translation["tr"]}} in English? + id: 07cfea8a-6d20-4719-a4ba-2ef8af3c416f + jinja: 'How do you say {{translation["en"]}} in Turkish? - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-tr-en-target + - how_to_say-en-tr-target reference: '' - 151edb16-4e30-4495-ab94-0506a3795f44: !Template + 0905d853-aa09-4e6b-a1ec-87a61bfbaabf: !Template answer_choices: null - id: 151edb16-4e30-4495-ab94-0506a3795f44 - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: 0905d853-aa09-4e6b-a1ec-87a61bfbaabf + jinja: '{{translation["tr"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 2cf4e3c4-a1f5-48c1-b87a-5625934d351f: !Template + - translate_as_tr-en-target + reference: '' + 0fe7d6a7-36d9-47ab-932d-b6a260f35893: !Template answer_choices: null - id: 2cf4e3c4-a1f5-48c1-b87a-5625934d351f - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: 0fe7d6a7-36d9-47ab-932d-b6a260f35893 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target + - version-tr-en-source+target reference: '' - 527c953c-a8b6-4876-b7d1-d7fe67edabbb: !Template + 161d9236-0bf4-48f3-896c-7814b8b516a3: !Template answer_choices: null - id: 527c953c-a8b6-4876-b7d1-d7fe67edabbb - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + id: 161d9236-0bf4-48f3-896c-7814b8b516a3 + jinja: 'Translate into Turkish: {{translation["en"]}} ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -70,27 +70,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-src+target + - translate-en-tr-target reference: '' - 5f09b297-0c8a-4d66-84cb-36e9cd19f9f9: !Template + 20e73f16-c5a1-4309-99a8-1f814c071b94: !Template answer_choices: null - id: 5f09b297-0c8a-4d66-84cb-36e9cd19f9f9 - jinja: '{{translation["tr"]}} translates into English as: + id: 20e73f16-c5a1-4309-99a8-1f814c071b94 + jinja: 'English: {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-target + - translate_as_en-tr-source+target reference: '' - 61637dbe-1c10-47f6-a466-ddb8f1964c06: !Template + 2ede56b7-1bc4-4020-9967-5c5cab76e969: !Template answer_choices: null - id: 61637dbe-1c10-47f6-a466-ddb8f1964c06 - jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version - should say: + id: 2ede56b7-1bc4-4020-9967-5c5cab76e969 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -99,12 +99,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-tr-en-source+target + - version-tr-en-target reference: '' - 63d5e992-f53e-4e9f-8205-923084e7cbeb: !Template + 314ad496-6be3-47db-98bf-69c8552d32e7: !Template answer_choices: null - id: 63d5e992-f53e-4e9f-8205-923084e7cbeb - jinja: 'What is the Turkish translation of: {{translation["en"]}} + id: 314ad496-6be3-47db-98bf-69c8552d32e7 + jinja: 'Translate English into Turkish: {{translation["en"]}} ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -113,13 +113,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target + - translate-en-tr-source+starget reference: '' - 779758cf-ed3e-412d-8862-2accc1bd8b4a: !Template + 3a2e1972-6f3c-4f85-8bc7-e4bd238ad974: !Template answer_choices: null - id: 779758cf-ed3e-412d-8862-2accc1bd8b4a - jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation - is: + id: 3a2e1972-6f3c-4f85-8bc7-e4bd238ad974 + jinja: 'Translate Turkish into English: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 42104024-1eba-41b1-abc5-09377e8b0ed9: !Template + answer_choices: null + id: 42104024-1eba-41b1-abc5-09377e8b0ed9 + jinja: 'Turkish: {{translation["tr"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -128,25 +141,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-target - reference: '' - 7e877011-b2a7-4413-8d9e-1c4ed886ad90: !Template + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4461db59-9557-4aef-b030-aedcfbe3e1b5: !Template answer_choices: null - id: 7e877011-b2a7-4413-8d9e-1c4ed886ad90 - jinja: 'Translate into Turkish: {{translation["en"]}} + id: 4461db59-9557-4aef-b030-aedcfbe3e1b5 + jinja: '{{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-target + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 45077a08-85e6-4986-a0e0-439a9bdfdde0: !Template + answer_choices: null + id: 45077a08-85e6-4986-a0e0-439a9bdfdde0 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_tr-en-source+target reference: '' - 7fbb5eea-a23f-4613-bb0d-e07d90d7b688: !Template + 45569b51-8855-4a84-9d01-7bd38861e88a: !Template answer_choices: null - id: 7fbb5eea-a23f-4613-bb0d-e07d90d7b688 + id: 45569b51-8855-4a84-9d01-7bd38861e88a jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -157,9 +184,9 @@ templates: name: !!python/tuple - a_good_translation-tr-en-source+target reference: '' - 8c33de3d-b2b8-49c7-b453-a37ccdb17fce: !Template + 59fd0965-c26d-49d4-bb0d-028e7299d9f4: !Template answer_choices: null - id: 8c33de3d-b2b8-49c7-b453-a37ccdb17fce + id: 59fd0965-c26d-49d4-bb0d-028e7299d9f4 jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version should say: @@ -172,52 +199,67 @@ templates: name: !!python/tuple - version-en-tr-source+target reference: '' - 8fc26430-12b6-425b-883d-cb9cc2d0535d: !Template + 5c3f1006-96be-4b01-8585-6982befd3b4c: !Template answer_choices: null - id: 8fc26430-12b6-425b-883d-cb9cc2d0535d - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: 5c3f1006-96be-4b01-8585-6982befd3b4c + jinja: ' {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target + - translate_as_en-tr-target reference: '' - a0912822-af17-4949-992e-5979481c82d0: !Template + 68d36e14-5f6e-4b9a-8a1a-373d56ab58e3: !Template answer_choices: null - id: a0912822-af17-4949-992e-5979481c82d0 - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: 68d36e14-5f6e-4b9a-8a1a-373d56ab58e3 + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-src+target + - a_good_translation-tr-en-target reference: '' - ad1a4010-fb06-4257-8789-4ea6550963a8: !Template + 7151e15d-1107-4ab3-b94a-98b0600546a4: !Template answer_choices: null - id: ad1a4010-fb06-4257-8789-4ea6550963a8 - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: 7151e15d-1107-4ab3-b94a-98b0600546a4 + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target - reference: '' - b1f8bdf0-1410-4de1-983b-aee71e1f202b: !Template + - gpt-3-en-tr-source+target + reference: GPT 3 paper + 9b22fa04-287e-457d-a530-60bfef922b5b: !Template answer_choices: null - id: b1f8bdf0-1410-4de1-983b-aee71e1f202b - jinja: ' {{translation["en"]}} translates into Turkish as: + id: 9b22fa04-287e-457d-a530-60bfef922b5b + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-source+target + reference: GPT-3 paper + 9de9346b-8c09-47be-8154-c2c1715e054e: !Template + answer_choices: null + id: 9de9346b-8c09-47be-8154-c2c1715e054e + jinja: '{{translation["en"]}} = Turkish: ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -226,39 +268,52 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target - reference: '' - be71bc64-4d1e-432b-ae21-344bf39b577a: !Template + - xglm-en-tr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c01ec82b-9c70-49b4-8244-ddd4ef89e963: !Template answer_choices: null - id: be71bc64-4d1e-432b-ae21-344bf39b577a - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: c01ec82b-9c70-49b4-8244-ddd4ef89e963 + jinja: 'English: {{translation["en"]}} = Turkish: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + ca5fa23d-8591-4d37-8a79-24103a52af45: !Template + answer_choices: null + id: ca5fa23d-8591-4d37-8a79-24103a52af45 + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-src+starget + - a_good_translation-en-tr-target reference: '' - d724d6f8-bc52-439f-898a-a32ea7317377: !Template + d7e68185-0fe7-41f1-b66b-9ad3e089a3fa: !Template answer_choices: null - id: d724d6f8-bc52-439f-898a-a32ea7317377 - jinja: 'What is the English translation of : {{translation["tr"]}} + id: d7e68185-0fe7-41f1-b66b-9ad3e089a3fa + jinja: 'What is the Turkish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target - reference: '' - d8bbecc3-f551-4931-aa64-d01d2b5d0630: !Template + - gpt-3-en-tr-target + reference: GPT-3 paper + dae5163c-ecaf-4eaa-805e-41a8f312eb05: !Template answer_choices: null - id: d8bbecc3-f551-4931-aa64-d01d2b5d0630 + id: dae5163c-ecaf-4eaa-805e-41a8f312eb05 jinja: 'Translate into English: {{translation["tr"]}} ||| {{translation["en"]}}' @@ -270,10 +325,10 @@ templates: name: !!python/tuple - translate-tr-en-target reference: '' - daad9f22-4bcf-41cb-9936-b17ff52a47d6: !Template + e4bfc8ed-3370-46d3-90be-187523d10e09: !Template answer_choices: null - id: daad9f22-4bcf-41cb-9936-b17ff52a47d6 - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: e4bfc8ed-3370-46d3-90be-187523d10e09 + jinja: 'How do you say {{translation["tr"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -282,34 +337,35 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target + - how_to_say-tr-en-target reference: '' - e8548697-38a5-4076-a11e-ac3be8f9ea48: !Template + e7c13937-1ced-4129-8de2-f51136e661c9: !Template answer_choices: null - id: e8548697-38a5-4076-a11e-ac3be8f9ea48 - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: e7c13937-1ced-4129-8de2-f51136e661c9 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target + - a_good_translation-en-tr-source+target reference: '' - f51b4cf6-bd87-4bbc-92f9-0f77b2834e48: !Template + f46b64ae-bbd4-449e-869c-c6c84a1d9aeb: !Template answer_choices: null - id: f51b4cf6-bd87-4bbc-92f9-0f77b2834e48 - jinja: 'How do you say {{translation["en"]}} in Turkish? + id: f46b64ae-bbd4-449e-869c-c6c84a1d9aeb + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - version-en-tr-target reference: '' diff --git a/promptsource/templates/wmt17/cs-en/templates.yaml b/promptsource/templates/wmt17/cs-en/templates.yaml index 471e828f1..d27ac0988 100644 --- a/promptsource/templates/wmt17/cs-en/templates.yaml +++ b/promptsource/templates/wmt17/cs-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt17 subset: cs-en templates: - 01a9b4cc-3e7d-4472-bc07-423d85faad52: !Template + 02a34024-126f-4147-92a4-85b919389b09: !Template answer_choices: null - id: 01a9b4cc-3e7d-4472-bc07-423d85faad52 + id: 02a34024-126f-4147-92a4-85b919389b09 jinja: 'Translate into Czech: {{translation["en"]}} ||| {{translation["cs"]}}' @@ -15,38 +15,37 @@ templates: name: !!python/tuple - translate-en-cs-target reference: '' - 0b196075-faee-4e54-be5c-28cde617bcec: !Template + 032b917b-7e01-4f7b-9826-48025d460e60: !Template answer_choices: null - id: 0b196075-faee-4e54-be5c-28cde617bcec - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: 032b917b-7e01-4f7b-9826-48025d460e60 + jinja: 'Translate into English: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - translate-cs-en-target reference: '' - 1deb5592-6867-421c-9421-e2a74a6ebf11: !Template + 05c1b0e4-2424-434d-9f40-c3d02c88a73b: !Template answer_choices: null - id: 1deb5592-6867-421c-9421-e2a74a6ebf11 - jinja: ' {{translation["en"]}} translates into Czech as: + id: 05c1b0e4-2424-434d-9f40-c3d02c88a73b + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target - reference: '' - 2aaea840-09fa-4496-9ac2-3ce92b739ccb: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 06e2c6b1-b6d0-4a05-bcec-004a2bfcf2f2: !Template answer_choices: null - id: 2aaea840-09fa-4496-9ac2-3ce92b739ccb + id: 06e2c6b1-b6d0-4a05-bcec-004a2bfcf2f2 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Czech translation is: @@ -59,67 +58,68 @@ templates: name: !!python/tuple - a_good_translation-en-cs-source+target reference: '' - 40e25397-0a8e-4289-b28a-aeaa1f23ddb2: !Template + 16492d65-0760-4b86-86b4-2578994dc9d1: !Template answer_choices: null - id: 40e25397-0a8e-4289-b28a-aeaa1f23ddb2 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: 16492d65-0760-4b86-86b4-2578994dc9d1 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target + - version-en-cs-source+target reference: '' - 59d1355f-c705-4267-95be-9e98bb5fbb90: !Template + 17937d5b-1d1e-4856-9f02-1c33d01e7613: !Template answer_choices: null - id: 59d1355f-c705-4267-95be-9e98bb5fbb90 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 17937d5b-1d1e-4856-9f02-1c33d01e7613 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - a_good_translation-cs-en-source+target reference: '' - 61056233-ad4c-4226-9854-9e4a46cfea3f: !Template + 1f5e3b84-8516-4c07-a9a0-52842a4a326b: !Template answer_choices: null - id: 61056233-ad4c-4226-9854-9e4a46cfea3f - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 1f5e3b84-8516-4c07-a9a0-52842a4a326b + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - version-cs-en-target reference: '' - 6bec36bf-a843-488c-8843-cae39b644efa: !Template + 35fde963-ed17-4cf4-99e9-050a7c501cd2: !Template answer_choices: null - id: 6bec36bf-a843-488c-8843-cae39b644efa - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version - should say: + id: 35fde963-ed17-4cf4-99e9-050a7c501cd2 + jinja: '{{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target - reference: '' - 6e96ba15-9c50-4770-a8df-d4d2e04885cd: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4d33c258-1566-47dd-81c0-1dab0499736d: !Template answer_choices: null - id: 6e96ba15-9c50-4770-a8df-d4d2e04885cd - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 4d33c258-1566-47dd-81c0-1dab0499736d + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -128,26 +128,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - version-cs-en-source+target reference: '' - 89c49a56-cb7a-4d19-bed9-a4ef79476cca: !Template + 5926a418-9730-4f3f-a913-16206f7db095: !Template answer_choices: null - id: 89c49a56-cb7a-4d19-bed9-a4ef79476cca - jinja: 'Translate into English: {{translation["cs"]}} + id: 5926a418-9730-4f3f-a913-16206f7db095 + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - how_to_say-en-cs-target reference: '' - 8ffa0081-e870-47e9-99cf-ef649372507b: !Template + 5ae66026-566f-4d69-80eb-98bfe9a748c5: !Template answer_choices: null - id: 8ffa0081-e870-47e9-99cf-ef649372507b - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 5ae66026-566f-4d69-80eb-98bfe9a748c5 + jinja: '{{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -156,110 +156,124 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target - reference: '' - 930e9528-823c-4d19-8a53-4d1a1534a06d: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5c5fc3e1-7125-4305-bbcb-1a28f18eb7d9: !Template answer_choices: null - id: 930e9528-823c-4d19-8a53-4d1a1534a06d - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: 5c5fc3e1-7125-4305-bbcb-1a28f18eb7d9 + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: '' - 9f3279de-22db-45e9-989b-22acebdf7f02: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 78702895-3efa-43db-9ecd-866d0b2146e9: !Template answer_choices: null - id: 9f3279de-22db-45e9-989b-22acebdf7f02 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 78702895-3efa-43db-9ecd-866d0b2146e9 + jinja: 'English: {{translation["en"]}} = Czech: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-src+starget - reference: '' - a5d42259-2578-4f0f-ab0d-be7c76122ed6: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 88a17d2a-d688-4d0e-8b28-a8b10e5ead48: !Template answer_choices: null - id: a5d42259-2578-4f0f-ab0d-be7c76122ed6 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 88a17d2a-d688-4d0e-8b28-a8b10e5ead48 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-src+target + - a_good_translation-cs-en-target reference: '' - ab56af3e-f85b-42e5-a4de-dd791093530d: !Template + 8e4a3e22-78a9-4fa7-94b4-1f9e9cf76467: !Template answer_choices: null - id: ab56af3e-f85b-42e5-a4de-dd791093530d - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: 8e4a3e22-78a9-4fa7-94b4-1f9e9cf76467 + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - translate_as_en-cs-source+target reference: '' - aca1630b-8e84-481f-a3a1-4bdaaaa6fcda: !Template + 9003b86f-7c59-459b-a5da-40f1e2ff5782: !Template answer_choices: null - id: aca1630b-8e84-481f-a3a1-4bdaaaa6fcda - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 9003b86f-7c59-459b-a5da-40f1e2ff5782 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-target + reference: '' + a0367ec8-1ff9-4d13-9d31-ce212285b2cc: !Template + answer_choices: null + id: a0367ec8-1ff9-4d13-9d31-ce212285b2cc + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - version-en-cs-target reference: '' - b847c014-264b-414a-9e84-1a087daefcb9: !Template + a625334b-415b-4fbb-9b93-2fd523d8d04a: !Template answer_choices: null - id: b847c014-264b-414a-9e84-1a087daefcb9 - jinja: '{{translation["cs"]}} translates into English as: + id: a625334b-415b-4fbb-9b93-2fd523d8d04a + jinja: ' {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - translate_as_en-cs-target reference: '' - ba823677-d7cd-4e42-9e55-120ee6b07e66: !Template + ab8f812b-5897-4062-bcfa-531d33948297: !Template answer_choices: null - id: ba823677-d7cd-4e42-9e55-120ee6b07e66 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: ab8f812b-5897-4062-bcfa-531d33948297 + jinja: 'Translate English into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-src+target + - translate-en-cs-source+starget reference: '' - bd8b3c35-291e-4b37-b270-c40234c11324: !Template + b56a417b-b38a-4ee8-b875-905985ca48b1: !Template answer_choices: null - id: bd8b3c35-291e-4b37-b270-c40234c11324 + id: b56a417b-b38a-4ee8-b875-905985ca48b1 jinja: 'Translate Czech into English: {{translation["cs"]}} ||| {{translation["en"]}}' @@ -271,23 +285,52 @@ templates: name: !!python/tuple - translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - e2a453da-ab63-4ce4-85fe-6173f3eff740: !Template + bb32fabf-50aa-4560-be6d-fd31947b9248: !Template answer_choices: null - id: e2a453da-ab63-4ce4-85fe-6173f3eff740 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: bb32fabf-50aa-4560-be6d-fd31947b9248 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - gpt-3-cs-en-target + reference: GPT-3 paper + beaf04b3-d4de-4ccf-a13d-f624ac1beaa1: !Template + answer_choices: null + id: beaf04b3-d4de-4ccf-a13d-f624ac1beaa1 + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_cs-en-source+target reference: '' - f98dbcb7-9da6-4903-893a-c2dd3337f2fe: !Template + d7376547-b699-4bc7-a753-7896092609eb: !Template answer_choices: null - id: f98dbcb7-9da6-4903-893a-c2dd3337f2fe - jinja: 'How do you say {{translation["cs"]}} in English? + id: d7376547-b699-4bc7-a753-7896092609eb + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-cs-source+target + reference: GPT 3 paper + f3b839ed-9535-4714-a10a-12436be1f517: !Template + answer_choices: null + id: f3b839ed-9535-4714-a10a-12436be1f517 + jinja: '{{translation["cs"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -296,20 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - translate_as_cs-en-target reference: '' - fed191b4-e6c1-4d6f-86dc-fcedbccadcf2: !Template + f764a6f3-bddd-4f63-9a63-faf18aab9c3e: !Template answer_choices: null - id: fed191b4-e6c1-4d6f-86dc-fcedbccadcf2 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: f764a6f3-bddd-4f63-9a63-faf18aab9c3e + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - gpt-3-cs-en-source+target + reference: GPT-3 paper + ff8c37a8-4b9f-4a12-8e91-46354e5720fb: !Template + answer_choices: null + id: ff8c37a8-4b9f-4a12-8e91-46354e5720fb + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-cs-en-target reference: '' diff --git a/promptsource/templates/wmt17/de-en/templates.yaml b/promptsource/templates/wmt17/de-en/templates.yaml index 3d966b2d9..4b9261304 100644 --- a/promptsource/templates/wmt17/de-en/templates.yaml +++ b/promptsource/templates/wmt17/de-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt17 subset: de-en templates: - 0412ba2f-a534-40bd-818f-9417b357eeea: !Template + 08524af5-6d00-4289-a1bb-935aa9af8666: !Template answer_choices: null - id: 0412ba2f-a534-40bd-818f-9417b357eeea - jinja: 'Translate German into English: {{translation["de"]}} + id: 08524af5-6d00-4289-a1bb-935aa9af8666 + jinja: 'What is the English translation of : {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -13,11 +13,24 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1497ff4a-f007-4761-a8f1-4b01ac2fdb3a: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + 22f21bc1-547f-46c5-8676-b8429d2b242f: !Template answer_choices: null - id: 1497ff4a-f007-4761-a8f1-4b01ac2fdb3a + id: 22f21bc1-547f-46c5-8676-b8429d2b242f + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + 279e6f95-4376-44f2-a4d4-a88320900c9f: !Template + answer_choices: null + id: 279e6f95-4376-44f2-a4d4-a88320900c9f jinja: 'How do you say {{translation["en"]}} in German? ||| {{translation["de"]}}' @@ -29,10 +42,40 @@ templates: name: !!python/tuple - how_to_say-en-de-target reference: '' - 28cee50e-f52b-4bc4-a295-c7ad574012b1: !Template + 2adfdaef-1e76-4faf-ac39-7a293180f238: !Template answer_choices: null - id: 28cee50e-f52b-4bc4-a295-c7ad574012b1 - jinja: '{{translation["de"]}} translates into English as: + id: 2adfdaef-1e76-4faf-ac39-7a293180f238 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-de-source+target + reference: '' + 33927585-3b79-4575-a14e-cc473d46f552: !Template + answer_choices: null + id: 33927585-3b79-4575-a14e-cc473d46f552 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-source+target + reference: '' + 465ea4d3-1368-4eaa-9a22-89ec5416ab49: !Template + answer_choices: null + id: 465ea4d3-1368-4eaa-9a22-89ec5416ab49 + jinja: 'German: {{translation["de"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -41,12 +84,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - translate_as_de-en-source+target reference: '' - 3788ad81-f204-4658-a5f5-95dd2273a0e5: !Template + 4f552a67-50f2-4ec1-8080-2d1fe90d27aa: !Template answer_choices: null - id: 3788ad81-f204-4658-a5f5-95dd2273a0e5 - jinja: 'Translate into German: {{translation["en"]}} + id: 4f552a67-50f2-4ec1-8080-2d1fe90d27aa + jinja: 'What is the German translation of: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -55,12 +98,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - gpt-3-en-de-target + reference: GPT-3 paper + 5789ef83-ee21-4277-9bc7-3c95fb8d2ee2: !Template + answer_choices: null + id: 5789ef83-ee21-4277-9bc7-3c95fb8d2ee2 + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-source+starget reference: '' - 4b443fc4-3f35-4420-ba69-9b3cc39c2377: !Template + 5e0b96c1-d9e7-4fee-b905-2977399c70f1: !Template answer_choices: null - id: 4b443fc4-3f35-4420-ba69-9b3cc39c2377 - jinja: 'What is the English translation of : {{translation["de"]}} + id: 5e0b96c1-d9e7-4fee-b905-2977399c70f1 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -69,11 +126,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: '' - 60e576b9-62f0-40b1-a55c-bbcdc9600ce8: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 6322d202-a8cb-4ffa-a425-99ae0e55a6ff: !Template answer_choices: null - id: 60e576b9-62f0-40b1-a55c-bbcdc9600ce8 + id: 6322d202-a8cb-4ffa-a425-99ae0e55a6ff jinja: 'If the German version says: {{translation["de"]}}; then the English version should say: @@ -86,11 +143,10 @@ templates: name: !!python/tuple - version-de-en-source+target reference: '' - 6e58dba7-4a9c-49f9-87cc-833941ffb953: !Template + 730615e3-353b-4388-b89f-9c99809b74cd: !Template answer_choices: null - id: 6e58dba7-4a9c-49f9-87cc-833941ffb953 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 730615e3-353b-4388-b89f-9c99809b74cd + jinja: '{{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -99,13 +155,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target - reference: '' - 807a15d9-c3f7-42c6-9081-242a67e5f36f: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 76a8f488-3ccd-4455-8083-cf985a4f1c5c: !Template answer_choices: null - id: 807a15d9-c3f7-42c6-9081-242a67e5f36f - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 76a8f488-3ccd-4455-8083-cf985a4f1c5c + jinja: '{{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -114,39 +169,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target - reference: '' - 883446e1-6d30-4279-87bd-f977e501d70f: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 78ac7c05-174a-4bb6-b41a-bff529679779: !Template answer_choices: null - id: 883446e1-6d30-4279-87bd-f977e501d70f - jinja: 'Translate English into German: {{translation["en"]}} + id: 78ac7c05-174a-4bb6-b41a-bff529679779 + jinja: ' {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget + - translate_as_en-de-target reference: '' - 8a2e744b-333a-4054-8d92-a344d236b57e: !Template + 889e5533-28c8-4094-9127-103d4741a341: !Template answer_choices: null - id: 8a2e744b-333a-4054-8d92-a344d236b57e - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 889e5533-28c8-4094-9127-103d4741a341 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target + - a_good_translation-de-en-target reference: '' - 8a35b90d-e148-44d5-b0d4-437af5c00ace: !Template + 8ef91b8e-9cda-4828-b26b-a474c03004b9: !Template answer_choices: null - id: 8a35b90d-e148-44d5-b0d4-437af5c00ace + id: 8ef91b8e-9cda-4828-b26b-a474c03004b9 jinja: 'Given the following source text in German: {{translation["de"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -157,124 +213,124 @@ templates: name: !!python/tuple - a_good_translation-de-en-source+target reference: '' - 8c78b498-8cec-44cc-a9fd-2ea72609f201: !Template + a4448a0c-f546-4332-83b6-fd78e415c0be: !Template answer_choices: null - id: 8c78b498-8cec-44cc-a9fd-2ea72609f201 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: a4448a0c-f546-4332-83b6-fd78e415c0be + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - translate_as_de-en-target reference: '' - 95667fc5-aaff-4675-828c-79be2badfee7: !Template + b3d26334-534b-4d2f-b244-eb6486081457: !Template answer_choices: null - id: 95667fc5-aaff-4675-828c-79be2badfee7 - jinja: 'German: {{translation["de"]}} translates into English as: + id: b3d26334-534b-4d2f-b244-eb6486081457 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target - reference: '' - a1a4e4f8-fec2-4802-b467-1a3110d9c732: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + b8bf6d86-a3be-4d00-ac77-e522b98b60c1: !Template answer_choices: null - id: a1a4e4f8-fec2-4802-b467-1a3110d9c732 - jinja: 'How do you say {{translation["de"]}} in English? + id: b8bf6d86-a3be-4d00-ac77-e522b98b60c1 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - version-de-en-target reference: '' - b78030c3-cef3-4aa8-ac05-ec753e24fdc3: !Template + c0c0eb2c-1fd1-42fc-b8de-7c2885d3eefb: !Template answer_choices: null - id: b78030c3-cef3-4aa8-ac05-ec753e24fdc3 - jinja: 'Translate into English: {{translation["de"]}} + id: c0c0eb2c-1fd1-42fc-b8de-7c2885d3eefb + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - version-en-de-target reference: '' - cb1bee80-252f-4bcb-8657-b1b063dd014b: !Template + c9f3b71f-77eb-4cb6-897a-257b8ac842da: !Template answer_choices: null - id: cb1bee80-252f-4bcb-8657-b1b063dd014b - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: c9f3b71f-77eb-4cb6-897a-257b8ac842da + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - how_to_say-de-en-target reference: '' - d350fec6-66c5-45f5-b9de-701faa4e37b8: !Template + e1a1c055-9dff-4a98-9a90-38338fbc0174: !Template answer_choices: null - id: d350fec6-66c5-45f5-b9de-701faa4e37b8 - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: e1a1c055-9dff-4a98-9a90-38338fbc0174 + jinja: 'Translate into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - translate-en-de-target reference: '' - e7a1e365-ac55-48b7-9c3f-a13551a29e0e: !Template + e5003513-1a97-41d4-ada8-15b9543e47b7: !Template answer_choices: null - id: e7a1e365-ac55-48b7-9c3f-a13551a29e0e - jinja: ' {{translation["en"]}} translates into German as: + id: e5003513-1a97-41d4-ada8-15b9543e47b7 + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target - reference: '' - ed7eba49-70b4-41ce-bb60-8ef299950dcf: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ea701736-3e22-4eb8-9bb1-182b12b54918: !Template answer_choices: null - id: ed7eba49-70b4-41ce-bb60-8ef299950dcf - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: ea701736-3e22-4eb8-9bb1-182b12b54918 + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - f0c3c5ae-cca6-434c-b54e-12c1b1440a69: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + efc189f5-ac49-4615-89f4-f343fb7b5e4c: !Template answer_choices: null - id: f0c3c5ae-cca6-434c-b54e-12c1b1440a69 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: efc189f5-ac49-4615-89f4-f343fb7b5e4c + jinja: 'Translate into English: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -283,33 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target + - translate-de-en-target reference: '' - f1febb5f-bb1f-4aad-aad8-fe1b89ee329b: !Template + f773be03-b0c4-439a-874f-00a58568c8b5: !Template answer_choices: null - id: f1febb5f-bb1f-4aad-aad8-fe1b89ee329b - jinja: 'English: {{translation["en"]}} translates into German as: + id: f773be03-b0c4-439a-874f-00a58568c8b5 + jinja: 'Translate German into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target - reference: '' - f6ff0677-5068-4e1f-940a-86ef559a464c: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + ffa3a920-0622-46ec-a234-017add93712c: !Template answer_choices: null - id: f6ff0677-5068-4e1f-940a-86ef559a464c - jinja: 'What is the German translation of: {{translation["en"]}} + id: ffa3a920-0622-46ec-a234-017add93712c + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target + - translate_as_en-de-source+target reference: '' diff --git a/promptsource/templates/wmt17/fi-en/templates.yaml b/promptsource/templates/wmt17/fi-en/templates.yaml index eac87c0ab..effc56b1d 100644 --- a/promptsource/templates/wmt17/fi-en/templates.yaml +++ b/promptsource/templates/wmt17/fi-en/templates.yaml @@ -1,66 +1,67 @@ dataset: wmt17 subset: fi-en templates: - 0088b4f4-fe6e-44b4-9740-62e258676a9a: !Template + 09d937c7-15dc-4699-9971-4900c643fc3b: !Template answer_choices: null - id: 0088b4f4-fe6e-44b4-9740-62e258676a9a - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: 09d937c7-15dc-4699-9971-4900c643fc3b + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target - reference: '' - 15909dca-c95f-4395-be59-f1c67498422e: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + 1030aa90-aa70-4d1f-a819-a44f278d13db: !Template answer_choices: null - id: 15909dca-c95f-4395-be59-f1c67498422e - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 1030aa90-aa70-4d1f-a819-a44f278d13db + jinja: '{{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-src+starget - reference: '' - 27078522-e6b8-419f-aab5-3f37484790ef: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 1a7e4164-f420-4845-b209-04d017592cb7: !Template answer_choices: null - id: 27078522-e6b8-419f-aab5-3f37484790ef - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 1a7e4164-f420-4845-b209-04d017592cb7 + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target - reference: '' - 2af8ac5d-cf62-4b4a-9ded-1dbc276bc406: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + 1b06cda4-384c-4e4f-848d-d2e4ecfbc0eb: !Template answer_choices: null - id: 2af8ac5d-cf62-4b4a-9ded-1dbc276bc406 - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 1b06cda4-384c-4e4f-848d-d2e4ecfbc0eb + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - translate_as_en-fi-source+target reference: '' - 2f61c246-5222-49d2-9e03-bd66cf249e76: !Template + 21f84cd6-a91c-42e6-b8ad-9d184b95bae1: !Template answer_choices: null - id: 2f61c246-5222-49d2-9e03-bd66cf249e76 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 21f84cd6-a91c-42e6-b8ad-9d184b95bae1 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -69,83 +70,95 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - version-fi-en-target reference: '' - 2fa36657-1938-4959-81e5-eca24f504804: !Template + 2606cfdf-7f82-4664-891a-326fc7fc0e6f: !Template answer_choices: null - id: 2fa36657-1938-4959-81e5-eca24f504804 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 2606cfdf-7f82-4664-891a-326fc7fc0e6f + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - how_to_say-fi-en-target reference: '' - 46a26437-fab2-4cc5-b7e2-968f29b1b606: !Template + 2edd1c4f-9555-4b8f-8e0b-a2bab6be396a: !Template answer_choices: null - id: 46a26437-fab2-4cc5-b7e2-968f29b1b606 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: + id: 2edd1c4f-9555-4b8f-8e0b-a2bab6be396a + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - translate_as_en-fi-target reference: '' - 48df8b43-caa8-4a08-84ed-b8508a134744: !Template + 3a1ccb5c-d1d4-49e4-a9d5-9e048b64dc66: !Template answer_choices: null - id: 48df8b43-caa8-4a08-84ed-b8508a134744 - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: 3a1ccb5c-d1d4-49e4-a9d5-9e048b64dc66 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target - reference: '' - 78571171-181d-43b5-aed4-c64dbbe0a88f: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 424c292d-22f8-4dbe-9f81-6c5dc6bb788c: !Template answer_choices: null - id: 78571171-181d-43b5-aed4-c64dbbe0a88f - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 424c292d-22f8-4dbe-9f81-6c5dc6bb788c + jinja: 'Translate Finnish into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target - reference: '' - 7c4fa3c5-5edd-4c08-ad9d-27a6149493b1: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 4ea3f2d1-3fd3-44fe-87cb-c2969b783433: !Template answer_choices: null - id: 7c4fa3c5-5edd-4c08-ad9d-27a6149493b1 - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: 4ea3f2d1-3fd3-44fe-87cb-c2969b783433 + jinja: 'What is the Finnish translation of: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - gpt-3-en-fi-target + reference: GPT-3 paper + 62b47c3d-8021-4160-8874-3eddb54cd621: !Template + answer_choices: null + id: 62b47c3d-8021-4160-8874-3eddb54cd621 + jinja: 'Translate into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-fi-target reference: '' - 83347cec-5e58-4242-b747-8ada796193a2: !Template + 6a54d46e-2823-4946-b387-2281a0de107a: !Template answer_choices: null - id: 83347cec-5e58-4242-b747-8ada796193a2 + id: 6a54d46e-2823-4946-b387-2281a0de107a jinja: '{{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' @@ -157,122 +170,137 @@ templates: name: !!python/tuple - translate_as_fi-en-target reference: '' - 8783347e-1618-4692-8cc3-09e382b9048c: !Template + 70707354-7e23-4c53-99c3-d2b7151ac6c5: !Template answer_choices: null - id: 8783347e-1618-4692-8cc3-09e382b9048c - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: 70707354-7e23-4c53-99c3-d2b7151ac6c5 + jinja: 'Translate into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-src+target + - translate-fi-en-target reference: '' - 90ebc53b-7099-48cd-8682-ea591c7b7270: !Template + 8b8ee16f-42e9-4253-b128-269d9191b130: !Template answer_choices: null - id: 90ebc53b-7099-48cd-8682-ea591c7b7270 - jinja: 'How do you say {{translation["en"]}} in Finnish? - - ||| {{translation["fi"]}}' + id: 8b8ee16f-42e9-4253-b128-269d9191b130 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - a_good_translation-en-fi-target reference: '' - 96b60ec8-4385-466c-a9b3-b3f0b6bcedc5: !Template + 8ca1100e-4741-4bad-bf26-c5af06f35eb7: !Template answer_choices: null - id: 96b60ec8-4385-466c-a9b3-b3f0b6bcedc5 - jinja: 'How do you say {{translation["fi"]}} in English? + id: 8ca1100e-4741-4bad-bf26-c5af06f35eb7 + jinja: 'Translate English into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - translate-en-fi-source+starget reference: '' - 9876be1e-30e4-42ae-a9cc-d6e657dd2fa4: !Template + b71c4561-eea1-4b01-a4a3-d795e32796c5: !Template answer_choices: null - id: 9876be1e-30e4-42ae-a9cc-d6e657dd2fa4 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: b71c4561-eea1-4b01-a4a3-d795e32796c5 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target + - version-fi-en-source+target reference: '' - a3d06482-31e6-4f96-9531-f312236e3ead: !Template + c0f12c08-16cf-4f04-bf21-aea1b895fbf4: !Template answer_choices: null - id: a3d06482-31e6-4f96-9531-f312236e3ead - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - - ||| {{translation["en"]}}' + id: c0f12c08-16cf-4f04-bf21-aea1b895fbf4 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-src+target + - a_good_translation-fi-en-source+target reference: '' - b0442673-6b81-44cf-ab0a-d2a922873a16: !Template + c42e3493-bfaa-4143-9a78-ee4a09785bcd: !Template answer_choices: null - id: b0442673-6b81-44cf-ab0a-d2a922873a16 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: c42e3493-bfaa-4143-9a78-ee4a09785bcd + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - c921267d-4041-442e-823d-340386da1d4e: !Template + - version-en-fi-target + reference: '' + c5ab6e25-8318-46b3-8505-db0bb87ddf0a: !Template answer_choices: null - id: c921267d-4041-442e-823d-340386da1d4e - jinja: 'Translate into English: {{translation["fi"]}} + id: c5ab6e25-8318-46b3-8505-db0bb87ddf0a + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - a_good_translation-fi-en-target reference: '' - f1790f28-f93b-43d6-ba9d-8118bb6e5330: !Template + c5dde640-4a2c-46ca-bff1-20ac8e506697: !Template answer_choices: null - id: f1790f28-f93b-43d6-ba9d-8118bb6e5330 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: c5dde640-4a2c-46ca-bff1-20ac8e506697 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + dcb7eee2-776a-43fd-b41e-047875c89b32: !Template + answer_choices: null + id: dcb7eee2-776a-43fd-b41e-047875c89b32 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-source+target reference: '' - f2590e75-f5ff-48fc-a01c-4694163ecdfd: !Template + ddc72d31-5ec6-4d4f-b844-14014b4214eb: !Template answer_choices: null - id: f2590e75-f5ff-48fc-a01c-4694163ecdfd - jinja: 'What is the English translation of : {{translation["fi"]}} + id: ddc72d31-5ec6-4d4f-b844-14014b4214eb + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -281,26 +309,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: '' - f7ca1c96-3a6d-4b76-843c-902658308396: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + e6e2abd5-6778-46b4-8406-a3c267999ff9: !Template answer_choices: null - id: f7ca1c96-3a6d-4b76-843c-902658308396 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: e6e2abd5-6778-46b4-8406-a3c267999ff9 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - how_to_say-en-fi-target reference: '' - ff0bff92-8f1b-422c-9f4a-84e5d5808301: !Template + eae9555c-c7a8-45ed-9061-efb510c06d84: !Template answer_choices: null - id: ff0bff92-8f1b-422c-9f4a-84e5d5808301 + id: eae9555c-c7a8-45ed-9061-efb510c06d84 jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version should say: @@ -313,3 +340,32 @@ templates: name: !!python/tuple - version-en-fi-source+target reference: '' + f0af9fb5-b096-41af-bd4c-6517c1e20c66: !Template + answer_choices: null + id: f0af9fb5-b096-41af-bd4c-6517c1e20c66 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-source+target + reference: '' + fefbc6f1-438a-43d5-aad7-07f7cdb96c55: !Template + answer_choices: null + id: fefbc6f1-438a-43d5-aad7-07f7cdb96c55 + jinja: 'English: {{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt17/lv-en/templates.yaml b/promptsource/templates/wmt17/lv-en/templates.yaml index f5138850a..3ed2015dd 100644 --- a/promptsource/templates/wmt17/lv-en/templates.yaml +++ b/promptsource/templates/wmt17/lv-en/templates.yaml @@ -1,39 +1,67 @@ dataset: wmt17 subset: lv-en templates: - 009646b7-7723-4d3b-b766-b4fe191b3bdd: !Template + 08b25640-bb2b-4521-bd62-cd97e4246ed3: !Template answer_choices: null - id: 009646b7-7723-4d3b-b766-b4fe191b3bdd - jinja: 'If the original version says: {{translation["lv"]}}; then the English - version should say: + id: 08b25640-bb2b-4521-bd62-cd97e4246ed3 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Latvian translation is: - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lv-en-target + - a_good_translation-en-lv-source+target reference: '' - 07f78f8d-e5f6-41ea-a48d-237f4c917990: !Template + 10e91e3c-0b78-4c82-8d1b-44678b8aed83: !Template answer_choices: null - id: 07f78f8d-e5f6-41ea-a48d-237f4c917990 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Latvian translation is: + id: 10e91e3c-0b78-4c82-8d1b-44678b8aed83 + jinja: 'Translate English into Latvian: {{translation["en"]}} - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lv-source+target + - translate-en-lv-source+starget + reference: '' + 19b5b477-e5db-4a2c-a11b-afbb59e69f25: !Template + answer_choices: null + id: 19b5b477-e5db-4a2c-a11b-afbb59e69f25 + jinja: 'Latvian: {{translation["lv"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lv-en-source+target + reference: '' + 1c81d65c-1b56-4e2d-a8f2-c230c1e9f0d5: !Template + answer_choices: null + id: 1c81d65c-1b56-4e2d-a8f2-c230c1e9f0d5 + jinja: 'If the English version says: {{translation["en"]}}; then the Latvian version + should say: + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-lv-source+target reference: '' - 1997ff46-074f-4154-bcd7-5131ea1ed7d0: !Template + 1d87c64f-f016-4437-95cd-f385869a121b: !Template answer_choices: null - id: 1997ff46-074f-4154-bcd7-5131ea1ed7d0 + id: 1d87c64f-f016-4437-95cd-f385869a121b jinja: 'If the original version says: {{translation["en"]}}; then the Latvian version should say: @@ -46,37 +74,38 @@ templates: name: !!python/tuple - version-en-lv-target reference: '' - 4b504847-a119-4d66-b3c6-4584c9f2e74b: !Template + 2b42a4aa-66c5-49e2-93f6-15250e2ab546: !Template answer_choices: null - id: 4b504847-a119-4d66-b3c6-4584c9f2e74b - jinja: 'What is the English translation of : {{translation["lv"]}} + id: 2b42a4aa-66c5-49e2-93f6-15250e2ab546 + jinja: 'What is the Latvian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lv-en-target - reference: '' - 630b8b7f-2335-4b2d-9c21-936a7a5597c4: !Template + - gpt-3-en-lv-source+target + reference: GPT 3 paper + 35e1ebe3-6eb5-4fe2-871c-a722331ea2d5: !Template answer_choices: null - id: 630b8b7f-2335-4b2d-9c21-936a7a5597c4 - jinja: 'What is the Latvian translation of: {{translation["en"]}} + id: 35e1ebe3-6eb5-4fe2-871c-a722331ea2d5 + jinja: 'If the original version says: {{translation["lv"]}}; then the English + version should say: - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lv-target + - version-lv-en-target reference: '' - 6e911898-76f2-4686-bd7f-8db625bc3d57: !Template + 4564679c-9033-4791-8bde-5b215ba17f63: !Template answer_choices: null - id: 6e911898-76f2-4686-bd7f-8db625bc3d57 + id: 4564679c-9033-4791-8bde-5b215ba17f63 jinja: 'Given the following passage: {{translation["lv"]}} , a good English translation is: @@ -89,65 +118,65 @@ templates: name: !!python/tuple - a_good_translation-lv-en-target reference: '' - 788254a0-c30d-4cf7-997d-b4af2b6c3f74: !Template + 4a16394d-aee6-4df5-8704-d8854294e79f: !Template answer_choices: null - id: 788254a0-c30d-4cf7-997d-b4af2b6c3f74 - jinja: 'Given the following passage: {{translation["en"]}} , a good Latvian translation - is: ||| {{translation["lv"]}}' + id: 4a16394d-aee6-4df5-8704-d8854294e79f + jinja: '{{translation["en"]}} = Latvian: + + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lv-target - reference: '' - 7b9214a6-7bb2-4965-82dc-26c878b6b14f: !Template + - xglm-en-lv-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4b9dceb5-49dd-4354-a1b2-57059bb0b1b2: !Template answer_choices: null - id: 7b9214a6-7bb2-4965-82dc-26c878b6b14f - jinja: 'Translate into Latvian: {{translation["en"]}} + id: 4b9dceb5-49dd-4354-a1b2-57059bb0b1b2 + jinja: ' {{translation["en"]}} translates into Latvian as: - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lv-target + - translate_as_en-lv-target reference: '' - 7eca2651-caca-4d2e-825d-b5c580d09a62: !Template + 61290165-dec6-437b-80c4-ec4350b11102: !Template answer_choices: null - id: 7eca2651-caca-4d2e-825d-b5c580d09a62 - jinja: 'If the Latvian version says: {{translation["lv"]}}; then the English version - should say: + id: 61290165-dec6-437b-80c4-ec4350b11102 + jinja: 'Translate into English: {{translation["lv"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lv-en-source+target + - translate-lv-en-target reference: '' - 973aab0a-a230-497f-8f4f-e41adba60a97: !Template + 664cf31e-17a3-4ad5-92f5-5d38063b25f6: !Template answer_choices: null - id: 973aab0a-a230-497f-8f4f-e41adba60a97 - jinja: ' {{translation["en"]}} translates into Latvian as: + id: 664cf31e-17a3-4ad5-92f5-5d38063b25f6 + jinja: 'What is the English translation of : {{translation["lv"]}} - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lv-target - reference: '' - a549c27c-acee-471e-bb6e-926f4eb780d8: !Template + - gpt-3-lv-en-target + reference: GPT-3 paper + 78a53869-2df4-45c0-9506-a640eaf2073c: !Template answer_choices: null - id: a549c27c-acee-471e-bb6e-926f4eb780d8 + id: 78a53869-2df4-45c0-9506-a640eaf2073c jinja: '{{translation["lv"]}} translates into English as: ||| {{translation["en"]}}' @@ -159,122 +188,136 @@ templates: name: !!python/tuple - translate_as_lv-en-target reference: '' - ae4d88ef-7fa4-49f2-ab77-36a95b28d3d1: !Template + 810d05bf-6851-42d4-a62a-80a3a2b34066: !Template answer_choices: null - id: ae4d88ef-7fa4-49f2-ab77-36a95b28d3d1 - jinja: 'English: {{translation["en"]}} translates into Latvian as: + id: 810d05bf-6851-42d4-a62a-80a3a2b34066 + jinja: '{{translation["lv"]}} = English: - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lv-source+target + - xglm-lv-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 84586c6e-9f21-4219-aaef-acd3e580964c: !Template + answer_choices: null + id: 84586c6e-9f21-4219-aaef-acd3e580964c + jinja: 'Translate into Latvian: {{translation["en"]}} + + ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-lv-target reference: '' - aff14994-4e76-44fa-ad8e-f2ca5959ff39: !Template + 894c74f2-7e1c-41c7-9fd8-f21c0931ce38: !Template answer_choices: null - id: aff14994-4e76-44fa-ad8e-f2ca5959ff39 - jinja: 'How do you say {{translation["lv"]}} in English? + id: 894c74f2-7e1c-41c7-9fd8-f21c0931ce38 + jinja: 'If the Latvian version says: {{translation["lv"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-lv-en-target + - version-lv-en-source+target reference: '' - b23244c9-21ee-4c6f-83c7-1c2a2c75e1a0: !Template + 9536897d-9ebc-4340-bba3-7956d9ac60a1: !Template answer_choices: null - id: b23244c9-21ee-4c6f-83c7-1c2a2c75e1a0 - jinja: 'Translate Latvian into English: {{translation["lv"]}} + id: 9536897d-9ebc-4340-bba3-7956d9ac60a1 + jinja: 'What is the Latvian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lv-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - c24231a3-1294-4ea8-8bb6-789b6fc60868: !Template + - gpt-3-en-lv-target + reference: GPT-3 paper + 9dae3dbd-3828-4589-8a0f-c69af4b724b3: !Template answer_choices: null - id: c24231a3-1294-4ea8-8bb6-789b6fc60868 - jinja: 'Given the following source text in Latvian: {{translation["lv"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 9dae3dbd-3828-4589-8a0f-c69af4b724b3 + jinja: 'English: {{translation["en"]}} = Latvian: + + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lv-en-source+target - reference: '' - ceb16807-043a-4298-8766-0f60b710b364: !Template + - xglm-en-lv-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + a334cdf7-eedf-4222-bc93-cde54276f636: !Template answer_choices: null - id: ceb16807-043a-4298-8766-0f60b710b364 - jinja: 'If the English version says: {{translation["en"]}}; then the Latvian version - should say: + id: a334cdf7-eedf-4222-bc93-cde54276f636 + jinja: 'Translate Latvian into English: {{translation["lv"]}} - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lv-source+target - reference: '' - d065a3f5-21ef-4f64-b2a1-93be707dadcd: !Template + - translate-lv-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + bf88be1e-ffdb-4544-a205-69b9d7b4264d: !Template answer_choices: null - id: d065a3f5-21ef-4f64-b2a1-93be707dadcd - jinja: 'What is the Latvian translation of the English sentence: {{translation["en"]}} + id: bf88be1e-ffdb-4544-a205-69b9d7b4264d + jinja: 'What is the English translation of the Latvian sentence: {{translation["lv"]}} - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lv-src+target - reference: '' - d2acabb3-9a40-4481-8cc5-9be87ef0be1f: !Template + - gpt-3-lv-en-source+target + reference: GPT-3 paper + c3da2972-d194-4fd6-8080-31f10466e9ab: !Template answer_choices: null - id: d2acabb3-9a40-4481-8cc5-9be87ef0be1f - jinja: 'Translate English into Latvian: {{translation["en"]}} + id: c3da2972-d194-4fd6-8080-31f10466e9ab + jinja: 'How do you say {{translation["en"]}} in Latvian? - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lv-src+starget + - how_to_say-en-lv-target reference: '' - db6c79b7-4d5b-43e5-bd40-9a0a557cd67f: !Template + c8fc6565-7b8c-45b8-afd9-a48ca71cd6d8: !Template answer_choices: null - id: db6c79b7-4d5b-43e5-bd40-9a0a557cd67f - jinja: 'What is the English translation of the Latvian sentence: {{translation["lv"]}} - - ||| {{translation["en"]}}' + id: c8fc6565-7b8c-45b8-afd9-a48ca71cd6d8 + jinja: 'Given the following source text in Latvian: {{translation["lv"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lv-en-src+target + - a_good_translation-lv-en-source+target reference: '' - dbdb9686-a851-46bf-8364-bf85eea6d5f0: !Template + ce70bac8-4e16-482b-b211-974f455a86b7: !Template answer_choices: null - id: dbdb9686-a851-46bf-8364-bf85eea6d5f0 - jinja: 'Latvian: {{translation["lv"]}} translates into English as: + id: ce70bac8-4e16-482b-b211-974f455a86b7 + jinja: 'Latvian: {{translation["lv"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -283,26 +326,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_lv-en-source+target + - xglm-lv-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d40be9ac-abae-4d6b-ba60-1fdac62fe5ba: !Template + answer_choices: null + id: d40be9ac-abae-4d6b-ba60-1fdac62fe5ba + jinja: 'Given the following passage: {{translation["en"]}} , a good Latvian translation + is: ||| {{translation["lv"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lv-target reference: '' - dde440d5-0458-4221-86b5-0315ef4505bb: !Template + e3e8d9d1-3b6d-4739-97aa-234bd0aa1261: !Template answer_choices: null - id: dde440d5-0458-4221-86b5-0315ef4505bb - jinja: 'Translate into English: {{translation["lv"]}} + id: e3e8d9d1-3b6d-4739-97aa-234bd0aa1261 + jinja: 'How do you say {{translation["lv"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lv-en-target + - how_to_say-lv-en-target reference: '' - fface4f5-aa4d-4ede-b395-08e20defe516: !Template + eb064e2e-da31-416e-b93a-47001fce3217: !Template answer_choices: null - id: fface4f5-aa4d-4ede-b395-08e20defe516 - jinja: 'How do you say {{translation["en"]}} in Latvian? + id: eb064e2e-da31-416e-b93a-47001fce3217 + jinja: 'English: {{translation["en"]}} translates into Latvian as: ||| {{translation["lv"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-lv-target + - translate_as_en-lv-source+target reference: '' diff --git a/promptsource/templates/wmt17/ru-en/templates.yaml b/promptsource/templates/wmt17/ru-en/templates.yaml index 7e223828e..3706fd030 100644 --- a/promptsource/templates/wmt17/ru-en/templates.yaml +++ b/promptsource/templates/wmt17/ru-en/templates.yaml @@ -1,38 +1,38 @@ dataset: wmt17 subset: ru-en templates: - 01f14ae4-4289-447a-b769-533bc6b2cfa4: !Template + 0607296e-2db7-4eaf-861a-3a9a9e804b6e: !Template answer_choices: null - id: 01f14ae4-4289-447a-b769-533bc6b2cfa4 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 0607296e-2db7-4eaf-861a-3a9a9e804b6e + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - translate_as_ru-en-target reference: '' - 0b70acb0-fc6c-44e7-93e4-2472c8fd9610: !Template + 0f39c75a-ea5b-4327-bb94-dbc98bfe53d2: !Template answer_choices: null - id: 0b70acb0-fc6c-44e7-93e4-2472c8fd9610 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 0f39c75a-ea5b-4327-bb94-dbc98bfe53d2 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - how_to_say-en-ru-target reference: '' - 0c082698-0676-4ce9-bd42-4985fc6ab3bc: !Template + 0f6bec96-40ff-4a8d-bb23-8b2e5dc4c45d: !Template answer_choices: null - id: 0c082698-0676-4ce9-bd42-4985fc6ab3bc - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 0f6bec96-40ff-4a8d-bb23-8b2e5dc4c45d + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -41,11 +41,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 119a637e-be90-4e0c-a020-bdfeeb2269a3: !Template + answer_choices: null + id: 119a637e-be90-4e0c-a020-bdfeeb2269a3 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target reference: '' - 0e9f93b2-0de5-4232-8486-01fc765ebc45: !Template + 122aab6c-3067-4605-9aa0-ae3c1e38be82: !Template answer_choices: null - id: 0e9f93b2-0de5-4232-8486-01fc765ebc45 + id: 122aab6c-3067-4605-9aa0-ae3c1e38be82 + jinja: 'Russian: {{translation["ru"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 232d0ee6-75f6-40af-a1f4-cfa601ef902f: !Template + answer_choices: null + id: 232d0ee6-75f6-40af-a1f4-cfa601ef902f jinja: 'Translate into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' @@ -57,10 +85,10 @@ templates: name: !!python/tuple - translate-en-ru-target reference: '' - 0ef81131-531c-43f9-89e8-1f237fdbee5b: !Template + 2e1aaad4-43a0-4e7d-bcaa-4a1b127fbc9e: !Template answer_choices: null - id: 0ef81131-531c-43f9-89e8-1f237fdbee5b - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 2e1aaad4-43a0-4e7d-bcaa-4a1b127fbc9e + jinja: 'What is the English translation of : {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -69,12 +97,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target - reference: '' - 1897c494-b380-45ef-b321-eee93ccc8e77: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 33421b68-3dcc-4cba-8dae-fb5748bde1b3: !Template answer_choices: null - id: 1897c494-b380-45ef-b321-eee93ccc8e77 - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 33421b68-3dcc-4cba-8dae-fb5748bde1b3 + jinja: 'Translate English into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -83,82 +111,86 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target + - translate-en-ru-source+starget reference: '' - 20ce6360-fdc0-40e6-9fa7-fe710a24f90a: !Template + 42ec8f17-2b4a-4519-96af-29e00085fb90: !Template answer_choices: null - id: 20ce6360-fdc0-40e6-9fa7-fe710a24f90a - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 42ec8f17-2b4a-4519-96af-29e00085fb90 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - a_good_translation-en-ru-source+target reference: '' - 47870e82-5ed7-4a76-a94d-f9530cf31dc1: !Template + 4471f796-01ca-4b57-b553-e7e5954f3b49: !Template answer_choices: null - id: 47870e82-5ed7-4a76-a94d-f9530cf31dc1 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 4471f796-01ca-4b57-b553-e7e5954f3b49 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - translate_as_en-ru-target reference: '' - 68e43b6a-ae41-4d30-b088-b438cf1305a2: !Template + 4cf94fd2-3ad5-46cd-ace5-5b4b5a20df9b: !Template answer_choices: null - id: 68e43b6a-ae41-4d30-b088-b438cf1305a2 - jinja: ' {{translation["en"]}} translates into Russian as: + id: 4cf94fd2-3ad5-46cd-ace5-5b4b5a20df9b + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - a_good_translation-ru-en-target reference: '' - 759f64ca-5f6c-4266-8af1-77e4cecb0003: !Template + 53709362-7f93-4e4b-956a-21c967c34319: !Template answer_choices: null - id: 759f64ca-5f6c-4266-8af1-77e4cecb0003 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 53709362-7f93-4e4b-956a-21c967c34319 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - version-ru-en-target reference: '' - 76d43586-31d4-4203-be38-e16d4a8d7ea6: !Template + 55297bd5-a3b1-4cdb-92d8-f72ffd58545e: !Template answer_choices: null - id: 76d43586-31d4-4203-be38-e16d4a8d7ea6 - jinja: 'Translate into English: {{translation["ru"]}} + id: 55297bd5-a3b1-4cdb-92d8-f72ffd58545e + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - version-ru-en-source+target reference: '' - 79ad759f-3888-4e17-a5aa-ab9060c1ba8b: !Template + 5d092fdc-0c1d-4a3f-b619-c62eff356499: !Template answer_choices: null - id: 79ad759f-3888-4e17-a5aa-ab9060c1ba8b - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 5d092fdc-0c1d-4a3f-b619-c62eff356499 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -167,26 +199,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 6b7cb084-c2cb-46ff-b432-72e754639497: !Template + answer_choices: null + id: 6b7cb084-c2cb-46ff-b432-72e754639497 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target reference: '' - 871fe390-9b79-4c26-b006-d1e75a2b4ba3: !Template + 78994540-0af7-4b62-b90b-48e63ece9f69: !Template answer_choices: null - id: 871fe390-9b79-4c26-b006-d1e75a2b4ba3 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 78994540-0af7-4b62-b90b-48e63ece9f69 + jinja: 'Translate into English: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - translate-ru-en-target reference: '' - 891fa5eb-33d7-4521-905c-aeaa1d4e0ca4: !Template + 933bf831-8fee-4852-8fdf-4cc59670aaa0: !Template answer_choices: null - id: 891fa5eb-33d7-4521-905c-aeaa1d4e0ca4 + id: 933bf831-8fee-4852-8fdf-4cc59670aaa0 jinja: 'How do you say {{translation["ru"]}} in English? ||| {{translation["en"]}}' @@ -198,53 +242,53 @@ templates: name: !!python/tuple - how_to_say-ru-en-target reference: '' - 8aca02d8-ce15-4698-96d7-afd2ba792cf8: !Template + 9a9e4d2b-0930-4940-ba9f-65bfcbcf835d: !Template answer_choices: null - id: 8aca02d8-ce15-4698-96d7-afd2ba792cf8 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 9a9e4d2b-0930-4940-ba9f-65bfcbcf835d + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - version-en-ru-target reference: '' - 9e9ffa6b-0938-450f-877c-8d68f36b34d4: !Template + 9b7040e1-2e67-4aaf-924f-ef69d08b3610: !Template answer_choices: null - id: 9e9ffa6b-0938-450f-877c-8d68f36b34d4 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: - - ||| {{translation["ru"]}}' + id: 9b7040e1-2e67-4aaf-924f-ef69d08b3610 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - a_good_translation-en-ru-target reference: '' - b36630d8-9c80-416b-a648-8eb47c897c0f: !Template + ad2c10e3-8361-4177-bc43-a1bd88cfd592: !Template answer_choices: null - id: b36630d8-9c80-416b-a648-8eb47c897c0f - jinja: '{{translation["ru"]}} translates into English as: + id: ad2c10e3-8361-4177-bc43-a1bd88cfd592 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - version-en-ru-source+target reference: '' - bdcb8bc3-c756-4db9-9f60-f810efc151ac: !Template + bac788df-2d7f-4133-a0a0-dac44d996160: !Template answer_choices: null - id: bdcb8bc3-c756-4db9-9f60-f810efc151ac - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: bac788df-2d7f-4133-a0a0-dac44d996160 + jinja: 'English: {{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -253,11 +297,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + c45eb61c-01be-45fe-b0fe-3f6c987a063f: !Template + answer_choices: null + id: c45eb61c-01be-45fe-b0fe-3f6c987a063f + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target reference: '' - d8bc3227-8fb9-4cb3-b252-257dcc1247fa: !Template + e1d0c98c-cb14-4b97-977a-b869fe475006: !Template answer_choices: null - id: d8bc3227-8fb9-4cb3-b252-257dcc1247fa + id: e1d0c98c-cb14-4b97-977a-b869fe475006 jinja: 'Translate Russian into English: {{translation["ru"]}} ||| {{translation["en"]}}' @@ -269,10 +327,10 @@ templates: name: !!python/tuple - translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - e74c996f-a680-4e8e-930b-d5b7fa2e0570: !Template + e82b2724-6d78-4013-a084-4e55102aecbe: !Template answer_choices: null - id: e74c996f-a680-4e8e-930b-d5b7fa2e0570 - jinja: 'How do you say {{translation["en"]}} in Russian? + id: e82b2724-6d78-4013-a084-4e55102aecbe + jinja: '{{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -281,35 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target - reference: '' - f2650a80-740f-4f71-b733-ee4653d906df: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f39520c0-750f-4436-b116-9fbc3ada77c6: !Template answer_choices: null - id: f2650a80-740f-4f71-b733-ee4653d906df - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: f39520c0-750f-4436-b116-9fbc3ada77c6 + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target - reference: '' - f6ab5cfa-db83-4452-8ef6-f6ddc1391a2d: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + fea25b62-6fb1-4512-8bba-cd9d88d83faf: !Template answer_choices: null - id: f6ab5cfa-db83-4452-8ef6-f6ddc1391a2d - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: fea25b62-6fb1-4512-8bba-cd9d88d83faf + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt17/tr-en/templates.yaml b/promptsource/templates/wmt17/tr-en/templates.yaml index 9d13647dd..fc09b2b06 100644 --- a/promptsource/templates/wmt17/tr-en/templates.yaml +++ b/promptsource/templates/wmt17/tr-en/templates.yaml @@ -1,24 +1,38 @@ dataset: wmt17 subset: tr-en templates: - 0698dd68-dd49-4466-aa1b-ce5af9299a33: !Template + 13e02590-6107-4130-973f-c1bde0b7761b: !Template answer_choices: null - id: 0698dd68-dd49-4466-aa1b-ce5af9299a33 - jinja: 'Translate into English: {{translation["tr"]}} + id: 13e02590-6107-4130-973f-c1bde0b7761b + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-target + - version-en-tr-source+target reference: '' - 0835de7e-0340-4606-99af-465918e70b63: !Template + 18fa9dc2-e7b6-4724-8d90-1c14b2228a28: !Template answer_choices: null - id: 0835de7e-0340-4606-99af-465918e70b63 - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + id: 18fa9dc2-e7b6-4724-8d90-1c14b2228a28 + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-target + reference: '' + 1efb1fbb-2b14-4923-9441-b91f79414748: !Template + answer_choices: null + id: 1efb1fbb-2b14-4923-9441-b91f79414748 + jinja: 'Translate English into Turkish: {{translation["en"]}} ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -27,11 +41,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-src+target + - translate-en-tr-source+starget reference: '' - 114e79e2-fc02-4af2-a360-10f060b55b73: !Template + 260eb8ab-1e49-434d-8c9a-1a4b32bec7c0: !Template answer_choices: null - id: 114e79e2-fc02-4af2-a360-10f060b55b73 + id: 260eb8ab-1e49-434d-8c9a-1a4b32bec7c0 jinja: 'Translate into Turkish: {{translation["en"]}} ||| {{translation["tr"]}}' @@ -43,9 +57,37 @@ templates: name: !!python/tuple - translate-en-tr-target reference: '' - 2784a23d-f6e5-466a-8514-1cc05faf45cf: !Template + 273cf0ae-35da-493a-9850-384b9171b131: !Template answer_choices: null - id: 2784a23d-f6e5-466a-8514-1cc05faf45cf + id: 273cf0ae-35da-493a-9850-384b9171b131 + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-source+target + reference: GPT 3 paper + 2990588c-38e5-476f-b0c5-feccda8b63da: !Template + answer_choices: null + id: 2990588c-38e5-476f-b0c5-feccda8b63da + jinja: 'What is the Turkish translation of: {{translation["en"]}} + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-tr-target + reference: GPT-3 paper + 2df1d1eb-efc4-4579-aa9a-40522ff6a765: !Template + answer_choices: null + id: 2df1d1eb-efc4-4579-aa9a-40522ff6a765 jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version should say: @@ -58,9 +100,9 @@ templates: name: !!python/tuple - version-tr-en-source+target reference: '' - 312a85f7-cd7f-4a8e-bb9a-1adcd3577799: !Template + 35a60dad-9931-4d04-b88d-c14832109928: !Template answer_choices: null - id: 312a85f7-cd7f-4a8e-bb9a-1adcd3577799 + id: 35a60dad-9931-4d04-b88d-c14832109928 jinja: '{{translation["tr"]}} translates into English as: ||| {{translation["en"]}}' @@ -72,52 +114,68 @@ templates: name: !!python/tuple - translate_as_tr-en-target reference: '' - 3158445b-99a2-4e94-8c59-7267865b6b7c: !Template + 39c782bb-130a-4d51-bc38-6f22e4993e78: !Template answer_choices: null - id: 3158445b-99a2-4e94-8c59-7267865b6b7c - jinja: 'How do you say {{translation["tr"]}} in English? + id: 39c782bb-130a-4d51-bc38-6f22e4993e78 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-tr-en-target + - a_good_translation-en-tr-source+target reference: '' - 348d9ba5-a8c7-4dd8-b6a9-521b624ea926: !Template + 44f6674b-965b-4bba-9b4e-b06588248248: !Template answer_choices: null - id: 348d9ba5-a8c7-4dd8-b6a9-521b624ea926 - jinja: ' {{translation["en"]}} translates into Turkish as: + id: 44f6674b-965b-4bba-9b4e-b06588248248 + jinja: 'Turkish: {{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target - reference: '' - 44706451-c997-472e-98a1-679b3385dd86: !Template + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 46aecf3a-1cfb-4736-b3bc-35eb6b69073e: !Template answer_choices: null - id: 44706451-c997-472e-98a1-679b3385dd86 - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: 46aecf3a-1cfb-4736-b3bc-35eb6b69073e + jinja: 'What is the English translation of : {{translation["tr"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target + - gpt-3-tr-en-target + reference: GPT-3 paper + 4f537521-2acd-4fa5-9fa4-491959ae801d: !Template + answer_choices: null + id: 4f537521-2acd-4fa5-9fa4-491959ae801d + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-tr-target reference: '' - 4b7c0209-2f7e-42e6-a650-32525a89005e: !Template + 5aab9b82-d8d6-4db5-a4c8-2c99dc86a822: !Template answer_choices: null - id: 4b7c0209-2f7e-42e6-a650-32525a89005e - jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation - is: + id: 5aab9b82-d8d6-4db5-a4c8-2c99dc86a822 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -126,55 +184,53 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-target + - translate_as_tr-en-source+target reference: '' - 54f7e78c-c0c7-4915-933f-0d5fc7421841: !Template + 6be83853-7be0-48f8-a3b2-a27a2caa33bb: !Template answer_choices: null - id: 54f7e78c-c0c7-4915-933f-0d5fc7421841 - jinja: 'English: {{translation["en"]}} translates into Turkish as: - - ||| {{translation["tr"]}}' + id: 6be83853-7be0-48f8-a3b2-a27a2caa33bb + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target + - a_good_translation-tr-en-source+target reference: '' - 6c5ca811-0f1e-4e88-a3ce-5440e402d7dc: !Template + 8a6bcf96-3f82-4673-973d-ff5b6b58ab7f: !Template answer_choices: null - id: 6c5ca811-0f1e-4e88-a3ce-5440e402d7dc - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: 8a6bcf96-3f82-4673-973d-ff5b6b58ab7f + jinja: ' {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target + - translate_as_en-tr-target reference: '' - 72bac994-4b07-4b1f-9745-a9dcd63e8fab: !Template + 9da207fe-b01f-4fb2-a526-2dca9e72c040: !Template answer_choices: null - id: 72bac994-4b07-4b1f-9745-a9dcd63e8fab - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: 9da207fe-b01f-4fb2-a526-2dca9e72c040 + jinja: 'How do you say {{translation["tr"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target + - how_to_say-tr-en-target reference: '' - 77051969-f618-442d-90eb-705623f5857e: !Template + 9f258684-4a9a-4fd8-bbe8-8cf7dd204191: !Template answer_choices: null - id: 77051969-f618-442d-90eb-705623f5857e - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: 9f258684-4a9a-4fd8-bbe8-8cf7dd204191 + jinja: 'Translate into English: {{translation["tr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -183,83 +239,83 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-src+target + - translate-tr-en-target reference: '' - 7943be2e-958b-4f14-864e-25c43ce4729e: !Template + a0c2b202-34e9-4776-9171-da21c84aacea: !Template answer_choices: null - id: 7943be2e-958b-4f14-864e-25c43ce4729e - jinja: 'What is the Turkish translation of: {{translation["en"]}} + id: a0c2b202-34e9-4776-9171-da21c84aacea + jinja: 'English: {{translation["en"]}} = Turkish: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target - reference: '' - a77a7de7-ac1a-4245-b52f-171edc9ec09c: !Template + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + ad67e0db-d79c-475b-8e3c-14f31b5214e2: !Template answer_choices: null - id: a77a7de7-ac1a-4245-b52f-171edc9ec09c - jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: ad67e0db-d79c-475b-8e3c-14f31b5214e2 + jinja: '{{translation["tr"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-source+target - reference: '' - adfe64cd-ae02-4932-9ebb-0b59b5501edb: !Template + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b6be2a76-b9f2-4d73-9971-c92d4416c19c: !Template answer_choices: null - id: adfe64cd-ae02-4932-9ebb-0b59b5501edb - jinja: 'How do you say {{translation["en"]}} in Turkish? + id: b6be2a76-b9f2-4d73-9971-c92d4416c19c + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - a_good_translation-tr-en-target reference: '' - c6f89166-46ed-4b58-bf48-7a2f7ba95ed3: !Template + b80d257a-cd9c-4114-a3bd-d9c28aa2ab68: !Template answer_choices: null - id: c6f89166-46ed-4b58-bf48-7a2f7ba95ed3 - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: b80d257a-cd9c-4114-a3bd-d9c28aa2ab68 + jinja: '{{translation["en"]}} = Turkish: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-src+starget - reference: '' - d0775d4e-9346-466e-a02c-7da052b2c5a3: !Template + - xglm-en-tr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d8bcf01b-6af0-4e6a-89a3-eb989436372f: !Template answer_choices: null - id: d0775d4e-9346-466e-a02c-7da052b2c5a3 - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: d8bcf01b-6af0-4e6a-89a3-eb989436372f + jinja: 'Translate Turkish into English: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target - reference: '' - d1f699a0-5f70-4bf6-8aa5-53d3b3e883d3: !Template + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e655f136-c707-4313-a860-e3ab883baee7: !Template answer_choices: null - id: d1f699a0-5f70-4bf6-8aa5-53d3b3e883d3 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: e655f136-c707-4313-a860-e3ab883baee7 + jinja: 'How do you say {{translation["en"]}} in Turkish? ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -268,41 +324,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target + - how_to_say-en-tr-target reference: '' - d2d816c6-4c8e-4bbf-8c54-68594c9a317d: !Template + ef26e5b8-fe37-4e05-b977-6dd89ac2a7b6: !Template answer_choices: null - id: d2d816c6-4c8e-4bbf-8c54-68594c9a317d - jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version - should say: + id: ef26e5b8-fe37-4e05-b977-6dd89ac2a7b6 + jinja: 'English: {{translation["en"]}} translates into Turkish as: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-source+target + - translate_as_en-tr-source+target reference: '' - ee687d87-9133-4921-900d-809a9060e1f7: !Template + f8ba7995-30e2-4b90-9e6c-da6058983479: !Template answer_choices: null - id: ee687d87-9133-4921-900d-809a9060e1f7 - jinja: 'What is the English translation of : {{translation["tr"]}} + id: f8ba7995-30e2-4b90-9e6c-da6058983479 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target + - version-tr-en-target reference: '' - fa047d18-35d3-4c75-b7ce-7b1078a9889c: !Template + fd327d86-96c0-4a0e-ab1c-ae1110398d11: !Template answer_choices: null - id: fa047d18-35d3-4c75-b7ce-7b1078a9889c - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: fd327d86-96c0-4a0e-ab1c-ae1110398d11 + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf + - gpt-3-tr-en-source+target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt17/zh-en/templates.yaml b/promptsource/templates/wmt17/zh-en/templates.yaml index 3fa2bb6f2..946b2e007 100644 --- a/promptsource/templates/wmt17/zh-en/templates.yaml +++ b/promptsource/templates/wmt17/zh-en/templates.yaml @@ -1,66 +1,52 @@ dataset: wmt17 subset: zh-en templates: - 052514b1-99d4-47f5-949f-df28d0dd5590: !Template + 056b54ef-6044-4fdc-997a-94e7d70c0ae6: !Template answer_choices: null - id: 052514b1-99d4-47f5-949f-df28d0dd5590 - jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version - should say: - - ||| {{translation["zh"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - version-en-zh-source+target - reference: '' - 0616df46-109a-4eb8-bd1a-075f0b85a145: !Template - answer_choices: null - id: 0616df46-109a-4eb8-bd1a-075f0b85a145 - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: 056b54ef-6044-4fdc-997a-94e7d70c0ae6 + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-src+target + - a_good_translation-zh-en-target reference: '' - 0f64087a-c73c-47a2-8380-6222bc75dfd0: !Template + 1183a3f9-7303-488b-9273-b92e4dc767f0: !Template answer_choices: null - id: 0f64087a-c73c-47a2-8380-6222bc75dfd0 - jinja: 'Translate Chinese into English: {{translation["zh"]}} + id: 1183a3f9-7303-488b-9273-b92e4dc767f0 + jinja: '{{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1a8e989a-f17c-4efb-8669-f175eb5c25a5: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 169b309d-5851-4444-a208-5bb91cde3af3: !Template answer_choices: null - id: 1a8e989a-f17c-4efb-8669-f175eb5c25a5 - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: 169b309d-5851-4444-a208-5bb91cde3af3 + jinja: 'How do you say {{translation["zh"]}} in English? - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target + - how_to_say-zh-en-target reference: '' - 1d82056d-254f-4df1-9b4b-332ce48faf9e: !Template + 1c810691-6c34-4e80-87ce-653ddb7330c9: !Template answer_choices: null - id: 1d82056d-254f-4df1-9b4b-332ce48faf9e + id: 1c810691-6c34-4e80-87ce-653ddb7330c9 jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -71,10 +57,10 @@ templates: name: !!python/tuple - a_good_translation-en-zh-target reference: '' - 2fe1c241-cdaf-4ff4-a7ac-78234251efba: !Template + 204091fc-5795-4dea-a245-0816eeb5c525: !Template answer_choices: null - id: 2fe1c241-cdaf-4ff4-a7ac-78234251efba - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: 204091fc-5795-4dea-a245-0816eeb5c525 + jinja: 'Chinese: {{translation["zh"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -83,25 +69,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target - reference: '' - 39c53426-27f6-45f4-8e51-268df51ba1e0: !Template + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 21880c27-4a9e-4676-a4c3-3685b4ea9d7d: !Template answer_choices: null - id: 39c53426-27f6-45f4-8e51-268df51ba1e0 - jinja: 'How do you say {{translation["zh"]}} in English? + id: 21880c27-4a9e-4676-a4c3-3685b4ea9d7d + jinja: 'English: {{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target - reference: '' - 49f79048-3ff5-405d-b4ef-9897db228826: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 30b7a571-9cb1-458f-bdb7-90eb442d0f15: !Template answer_choices: null - id: 49f79048-3ff5-405d-b4ef-9897db228826 + id: 30b7a571-9cb1-458f-bdb7-90eb442d0f15 jinja: 'Translate into English: {{translation["zh"]}} ||| {{translation["en"]}}' @@ -113,11 +99,25 @@ templates: name: !!python/tuple - translate-zh-en-target reference: '' - 4aea2385-5165-4df7-90bc-49a1e1affb70: !Template + 486bb7ac-1d82-4f8c-8795-0fdcbefdc44c: !Template answer_choices: null - id: 4aea2385-5165-4df7-90bc-49a1e1affb70 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Chinese translation is: + id: 486bb7ac-1d82-4f8c-8795-0fdcbefdc44c + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-source+target + reference: '' + 4b000b4b-64e0-4bc1-947d-3f7e967d8116: !Template + answer_choices: null + id: 4b000b4b-64e0-4bc1-947d-3f7e967d8116 + jinja: 'How do you say {{translation["en"]}} in Chinese? ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -126,11 +126,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-source+target + - how_to_say-en-zh-target reference: '' - 5cb6b5fb-ae44-4c4b-a59f-be09ca4232a8: !Template + 5821cf3f-41f2-4144-adc5-e6322e47e018: !Template answer_choices: null - id: 5cb6b5fb-ae44-4c4b-a59f-be09ca4232a8 + id: 5821cf3f-41f2-4144-adc5-e6322e47e018 jinja: ' {{translation["en"]}} translates into Chinese as: ||| {{translation["zh"]}}' @@ -142,24 +142,52 @@ templates: name: !!python/tuple - translate_as_en-zh-target reference: '' - 7735cfcf-592f-4451-be38-982880c17632: !Template + 5cdb4bca-fa23-4637-9459-02880508ec89: !Template answer_choices: null - id: 7735cfcf-592f-4451-be38-982880c17632 - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese - version should say: + id: 5cdb4bca-fa23-4637-9459-02880508ec89 + jinja: '{{translation["zh"]}} translates into English as: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target + - translate_as_zh-en-target + reference: '' + 7aff9be8-0ffd-47a1-aeb8-3c77daffcc33: !Template + answer_choices: null + id: 7aff9be8-0ffd-47a1-aeb8-3c77daffcc33 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-zh-en-source+target reference: '' - 97528e7b-a094-406e-9d51-42a073770288: !Template + 82897041-28e7-43d6-a9cc-4adf2599dda4: !Template + answer_choices: null + id: 82897041-28e7-43d6-a9cc-4adf2599dda4 + jinja: 'Translate Chinese into English: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 877c7d17-b3bc-4637-ac86-d0903d8bd4fe: !Template answer_choices: null - id: 97528e7b-a094-406e-9d51-42a073770288 + id: 877c7d17-b3bc-4637-ac86-d0903d8bd4fe jinja: 'What is the Chinese translation of: {{translation["en"]}} ||| {{translation["zh"]}}' @@ -170,10 +198,24 @@ templates: original_task: true name: !!python/tuple - gpt-3-en-zh-target - reference: '' - 9f156de8-375c-4b23-837e-0ca5d1f8a034: !Template + reference: GPT-3 paper + 8cdeef19-7f5d-477d-b4a7-6859f01fb581: !Template answer_choices: null - id: 9f156de8-375c-4b23-837e-0ca5d1f8a034 + id: 8cdeef19-7f5d-477d-b4a7-6859f01fb581 + jinja: '{{translation["zh"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a3d7aab6-d5a6-4a93-b516-bfe3a7147228: !Template + answer_choices: null + id: a3d7aab6-d5a6-4a93-b516-bfe3a7147228 jinja: 'Translate English into Chinese: {{translation["en"]}} ||| {{translation["zh"]}}' @@ -183,112 +225,125 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-src+starget + - translate-en-zh-source+starget reference: '' - ac364ea4-f761-4951-b971-e2b1766f4e94: !Template + ac3f9e03-e3bb-438a-b4e8-89e369dcc998: !Template answer_choices: null - id: ac364ea4-f761-4951-b971-e2b1766f4e94 - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: ac3f9e03-e3bb-438a-b4e8-89e369dcc998 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target + - a_good_translation-en-zh-source+target reference: '' - b455bed8-c1a6-4258-97b3-081c5f4c50f5: !Template + afb387ba-0dd6-4ba8-8b74-4fdeaeeaba46: !Template answer_choices: null - id: b455bed8-c1a6-4258-97b3-081c5f4c50f5 - jinja: 'Translate into Chinese: {{translation["en"]}} + id: afb387ba-0dd6-4ba8-8b74-4fdeaeeaba46 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - gpt-3-zh-en-source+target + reference: GPT-3 paper + c2ae991d-4700-433c-bf62-4cc0f69e7d35: !Template + answer_choices: null + id: c2ae991d-4700-433c-bf62-4cc0f69e7d35 + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-source+target reference: '' - bc317101-44e4-49fe-8f0f-07bee638a87a: !Template + c4d02e4e-1887-4ba9-8adc-80f7fe693ae2: !Template answer_choices: null - id: bc317101-44e4-49fe-8f0f-07bee638a87a - jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation - is: + id: c4d02e4e-1887-4ba9-8adc-80f7fe693ae2 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-target + - version-zh-en-target reference: '' - bede4c7b-a660-4d33-a5d3-c6fb29bd5d0d: !Template + c61d619c-6f9a-46be-bf7b-205f6f36ab00: !Template answer_choices: null - id: bede4c7b-a660-4d33-a5d3-c6fb29bd5d0d - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: c61d619c-6f9a-46be-bf7b-205f6f36ab00 + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target + - version-en-zh-target reference: '' - c0d710c2-3b3b-4c36-93b3-e8cedbb64748: !Template + c6c6cc52-2295-4782-8b34-c4d169cdc771: !Template answer_choices: null - id: c0d710c2-3b3b-4c36-93b3-e8cedbb64748 - jinja: '{{translation["zh"]}} translates into English as: + id: c6c6cc52-2295-4782-8b34-c4d169cdc771 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target - reference: '' - c7dcaeec-6c2e-4e8b-9fd2-fe145666777c: !Template + - gpt-3-en-zh-source+target + reference: GPT 3 paper + d8779d0f-5d92-4873-bb34-3782cf31d47d: !Template answer_choices: null - id: c7dcaeec-6c2e-4e8b-9fd2-fe145666777c - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: d8779d0f-5d92-4873-bb34-3782cf31d47d + jinja: 'English: {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target + - translate_as_en-zh-source+target reference: '' - cb4fe4e3-0e94-4563-aae9-846c0d170104: !Template + ea8d50d2-ffdf-4f3b-9f0a-fdb4dd9834ba: !Template answer_choices: null - id: cb4fe4e3-0e94-4563-aae9-846c0d170104 - jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + id: ea8d50d2-ffdf-4f3b-9f0a-fdb4dd9834ba + jinja: 'Chinese: {{translation["zh"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-src+target + - translate_as_zh-en-source+target reference: '' - e18a0dea-d7d1-4f77-a5a5-c27427d447ac: !Template + eed08d24-e22a-4a29-8336-fc58912dc806: !Template answer_choices: null - id: e18a0dea-d7d1-4f77-a5a5-c27427d447ac + id: eed08d24-e22a-4a29-8336-fc58912dc806 jinja: 'What is the English translation of : {{translation["zh"]}} ||| {{translation["en"]}}' @@ -299,17 +354,18 @@ templates: original_task: true name: !!python/tuple - gpt-3-zh-en-target - reference: '' - e571c8a2-57e1-45eb-84cb-8f31b6a1dfc6: !Template + reference: GPT-3 paper + fdbe9566-0482-4697-8d71-aae344ce4d5d: !Template answer_choices: null - id: e571c8a2-57e1-45eb-84cb-8f31b6a1dfc6 - jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: fdbe9566-0482-4697-8d71-aae344ce4d5d + jinja: 'Translate into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-source+target + - translate-en-zh-target reference: '' diff --git a/promptsource/templates/wmt18/cs-en/templates.yaml b/promptsource/templates/wmt18/cs-en/templates.yaml index 52cc86bf3..a0cd89cac 100644 --- a/promptsource/templates/wmt18/cs-en/templates.yaml +++ b/promptsource/templates/wmt18/cs-en/templates.yaml @@ -1,9 +1,37 @@ dataset: wmt18 subset: cs-en templates: - 040c8e39-deac-477f-a331-34630d0445ed: !Template + 04577e1a-ffdf-4b1e-83d2-b0e904710680: !Template answer_choices: null - id: 040c8e39-deac-477f-a331-34630d0445ed + id: 04577e1a-ffdf-4b1e-83d2-b0e904710680 + jinja: 'What is the English translation of : {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-target + reference: GPT-3 paper + 05776b12-0456-4600-966f-9faf823093ee: !Template + answer_choices: null + id: 05776b12-0456-4600-966f-9faf823093ee + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-cs-en-source+target + reference: GPT-3 paper + 058e1721-21a8-4ce0-a2bf-6880d2bf8e29: !Template + answer_choices: null + id: 058e1721-21a8-4ce0-a2bf-6880d2bf8e29 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Czech translation is: @@ -16,10 +44,10 @@ templates: name: !!python/tuple - a_good_translation-en-cs-source+target reference: '' - 04d83e59-cd90-471d-a47c-c28d8b5c4a3b: !Template + 05923706-77be-4421-9f2d-bd4508615e37: !Template answer_choices: null - id: 04d83e59-cd90-471d-a47c-c28d8b5c4a3b - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 05923706-77be-4421-9f2d-bd4508615e37 + jinja: ' {{translation["en"]}} translates into Czech as: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -28,26 +56,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_en-cs-target reference: '' - 061c1fe4-f3a4-4f2b-abd8-c9a3fb3024e1: !Template + 24252dfb-1e81-4c70-b482-f4c6efbbe6d3: !Template answer_choices: null - id: 061c1fe4-f3a4-4f2b-abd8-c9a3fb3024e1 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 24252dfb-1e81-4c70-b482-f4c6efbbe6d3 + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-src+starget + - how_to_say-en-cs-target reference: '' - 087fece4-a590-4668-b277-151e4106fc2e: !Template + 3e7e4a69-d624-4d0b-909b-2e212feafa75: !Template answer_choices: null - id: 087fece4-a590-4668-b277-151e4106fc2e - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + id: 3e7e4a69-d624-4d0b-909b-2e212feafa75 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version should say: ||| {{translation["cs"]}}' @@ -57,82 +85,80 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target + - version-en-cs-source+target reference: '' - 0a422243-290c-438a-84ec-970437fcaa1d: !Template + 49acd099-d766-48fc-8849-bb7620b6960d: !Template answer_choices: null - id: 0a422243-290c-438a-84ec-970437fcaa1d - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 49acd099-d766-48fc-8849-bb7620b6960d + jinja: 'How do you say {{translation["cs"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - how_to_say-cs-en-target reference: '' - 129856c6-778e-42a0-8b37-961952c9568a: !Template + 5e1da87f-b520-4ce6-9663-b22b546eed9c: !Template answer_choices: null - id: 129856c6-778e-42a0-8b37-961952c9568a - jinja: 'Translate into Czech: {{translation["en"]}} + id: 5e1da87f-b520-4ce6-9663-b22b546eed9c + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target - reference: '' - 1e43b70b-74b3-412d-b781-eab33e1f3539: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 7d3c501c-5c7c-4e54-84e1-97b24e90e673: !Template answer_choices: null - id: 1e43b70b-74b3-412d-b781-eab33e1f3539 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 7d3c501c-5c7c-4e54-84e1-97b24e90e673 + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-src+target + - translate_as_en-cs-source+target reference: '' - 1e92cbf9-f100-4af1-813e-28e04a5943e5: !Template + 806dc491-5193-42ce-a783-2801eef5b36b: !Template answer_choices: null - id: 1e92cbf9-f100-4af1-813e-28e04a5943e5 - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 806dc491-5193-42ce-a783-2801eef5b36b + jinja: 'Translate English into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - translate-en-cs-source+starget reference: '' - 21900f52-b821-4fce-8476-6ab56c2d5e7f: !Template + 86b123b4-5185-4dbe-9f49-30eb4b3181e0: !Template answer_choices: null - id: 21900f52-b821-4fce-8476-6ab56c2d5e7f - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: - - ||| {{translation["en"]}}' + id: 86b123b4-5185-4dbe-9f49-30eb4b3181e0 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - a_good_translation-cs-en-source+target reference: '' - 424b2603-bb66-4c35-8923-5bfe1a60a4a2: !Template + 92d63f28-f816-4432-a151-216f972297de: !Template answer_choices: null - id: 424b2603-bb66-4c35-8923-5bfe1a60a4a2 + id: 92d63f28-f816-4432-a151-216f972297de jinja: '{{translation["cs"]}} translates into English as: ||| {{translation["en"]}}' @@ -144,10 +170,10 @@ templates: name: !!python/tuple - translate_as_cs-en-target reference: '' - 5bb157ca-2812-4fad-bdb9-a44aaf4ae51f: !Template + 9953d68f-23d5-484f-a1c1-d714e2b529d4: !Template answer_choices: null - id: 5bb157ca-2812-4fad-bdb9-a44aaf4ae51f - jinja: 'What is the English translation of : {{translation["cs"]}} + id: 9953d68f-23d5-484f-a1c1-d714e2b529d4 + jinja: 'Translate Czech into English: {{translation["cs"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -156,125 +182,125 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target - reference: '' - 5ed29a0a-5deb-4506-ae64-34a77f2d338a: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + a33509f7-bd83-44c4-8335-8e05cd74c0f5: !Template answer_choices: null - id: 5ed29a0a-5deb-4506-ae64-34a77f2d338a - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: a33509f7-bd83-44c4-8335-8e05cd74c0f5 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target - reference: '' - 617e0505-47e2-479e-ad44-81c1a0aa1c6a: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + b074fa3f-a7ca-4d09-982d-28279d7567a7: !Template answer_choices: null - id: 617e0505-47e2-479e-ad44-81c1a0aa1c6a - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: b074fa3f-a7ca-4d09-982d-28279d7567a7 + jinja: '{{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target - reference: '' - 633165e3-840e-4436-ac11-0045d0eb7a52: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b0d50e7e-b106-4a3b-9ec1-d4d3e4a6be50: !Template answer_choices: null - id: 633165e3-840e-4436-ac11-0045d0eb7a52 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: b0d50e7e-b106-4a3b-9ec1-d4d3e4a6be50 + jinja: 'Translate into Czech: {{translation["en"]}} + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - translate-en-cs-target reference: '' - 731fefa1-3b89-4e01-8aec-b83e5a056366: !Template + b1a91890-7da9-45b5-a9e1-77a5b22411b8: !Template answer_choices: null - id: 731fefa1-3b89-4e01-8aec-b83e5a056366 - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: b1a91890-7da9-45b5-a9e1-77a5b22411b8 + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target - reference: '' - 7bda53a6-fc4d-4252-b747-f2a875de5768: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + b64106c0-74a9-4860-87ea-b1046f85601f: !Template answer_choices: null - id: 7bda53a6-fc4d-4252-b747-f2a875de5768 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: b64106c0-74a9-4860-87ea-b1046f85601f + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-src+target + - version-en-cs-target reference: '' - 7ff11297-f0b7-4539-98a5-c1da068a9f35: !Template + b742c3a5-1276-4992-a84f-f1f225d6f406: !Template answer_choices: null - id: 7ff11297-f0b7-4539-98a5-c1da068a9f35 - jinja: ' {{translation["en"]}} translates into Czech as: - - ||| {{translation["cs"]}}' + id: b742c3a5-1276-4992-a84f-f1f225d6f406 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - a_good_translation-en-cs-target reference: '' - 93aa6697-ca14-4945-bfe8-312d75d34693: !Template + c4462899-417c-4975-9dbe-e63a6de9f340: !Template answer_choices: null - id: 93aa6697-ca14-4945-bfe8-312d75d34693 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: c4462899-417c-4975-9dbe-e63a6de9f340 + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - b9851c1d-6c3c-44be-ab35-36a81d1e0a28: !Template + - translate_as_cs-en-source+target + reference: '' + cc1a6c32-2996-4106-92f8-50ddb3b67c4d: !Template answer_choices: null - id: b9851c1d-6c3c-44be-ab35-36a81d1e0a28 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: cc1a6c32-2996-4106-92f8-50ddb3b67c4d + jinja: '{{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target - reference: '' - bb1f8b1b-4bcf-4de4-a7bc-d90c7a69c6d8: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d434a2cd-8da4-4099-9a5d-f213adb25f0a: !Template answer_choices: null - id: bb1f8b1b-4bcf-4de4-a7bc-d90c7a69c6d8 - jinja: 'How do you say {{translation["cs"]}} in English? + id: d434a2cd-8da4-4099-9a5d-f213adb25f0a + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -283,25 +309,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - a_good_translation-cs-en-target reference: '' - cf9d7265-5b1d-454f-9c31-9cdc19c65fb0: !Template + da493ed1-90a4-4611-a423-a86610ffae8f: !Template answer_choices: null - id: cf9d7265-5b1d-454f-9c31-9cdc19c65fb0 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: da493ed1-90a4-4611-a423-a86610ffae8f + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target + - version-cs-en-source+target reference: '' - fa4118ef-a7ec-4a2e-8176-b4c28d67b7b7: !Template + e28156c4-1a78-4240-b8f7-1c89dee1e938: !Template answer_choices: null - id: fa4118ef-a7ec-4a2e-8176-b4c28d67b7b7 + id: e28156c4-1a78-4240-b8f7-1c89dee1e938 jinja: 'Translate into English: {{translation["cs"]}} ||| {{translation["en"]}}' @@ -313,3 +340,32 @@ templates: name: !!python/tuple - translate-cs-en-target reference: '' + f1f6a13a-3038-4e85-abfb-e110ef02a2b2: !Template + answer_choices: null + id: f1f6a13a-3038-4e85-abfb-e110ef02a2b2 + jinja: 'English: {{translation["en"]}} = Czech: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + f804109f-5858-4d43-868b-d5d6304c8146: !Template + answer_choices: null + id: f804109f-5858-4d43-868b-d5d6304c8146 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-cs-en-target + reference: '' diff --git a/promptsource/templates/wmt18/de-en/templates.yaml b/promptsource/templates/wmt18/de-en/templates.yaml index 82452fcc2..3c1007ec8 100644 --- a/promptsource/templates/wmt18/de-en/templates.yaml +++ b/promptsource/templates/wmt18/de-en/templates.yaml @@ -1,192 +1,209 @@ dataset: wmt18 subset: de-en templates: - 067b6cc5-c06c-4069-be26-6c4ba2acec6a: !Template + 07ad9f5e-0c3e-4eec-9381-533cfe063087: !Template answer_choices: null - id: 067b6cc5-c06c-4069-be26-6c4ba2acec6a - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: 07ad9f5e-0c3e-4eec-9381-533cfe063087 + jinja: 'German: {{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - translate_as_de-en-source+target reference: '' - 194a01fb-e73c-46a8-876f-c5e7a31bc306: !Template + 0a9966ee-ce22-464e-bb0e-3353d357fde4: !Template answer_choices: null - id: 194a01fb-e73c-46a8-876f-c5e7a31bc306 - jinja: 'Translate into English: {{translation["de"]}} + id: 0a9966ee-ce22-464e-bb0e-3353d357fde4 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - a_good_translation-en-de-source+target reference: '' - 22da89ee-3174-4828-b453-1c4d8e6c1542: !Template + 17e768cb-2d12-458b-822c-8b3566e0b512: !Template answer_choices: null - id: 22da89ee-3174-4828-b453-1c4d8e6c1542 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 17e768cb-2d12-458b-822c-8b3566e0b512 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - version-en-de-target reference: '' - 266a6865-d5e2-4400-a480-2920fc8a68df: !Template + 21963372-77d8-46f6-9514-ea839e998a17: !Template answer_choices: null - id: 266a6865-d5e2-4400-a480-2920fc8a68df - jinja: 'What is the English translation of : {{translation["de"]}} + id: 21963372-77d8-46f6-9514-ea839e998a17 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target + - how_to_say-en-de-target reference: '' - 2c9e97db-2782-4afd-9a91-abb7e1477388: !Template + 260e9766-9de6-4c2c-85a4-6b803b18ceaf: !Template answer_choices: null - id: 2c9e97db-2782-4afd-9a91-abb7e1477388 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 260e9766-9de6-4c2c-85a4-6b803b18ceaf + jinja: 'Translate into German: {{translation["en"]}} + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target + - translate-en-de-target reference: '' - 3fa7947d-3e69-48c6-a1ad-780ab6a50e9f: !Template + 285711ca-7f22-4e81-b56a-d909f3bd0467: !Template answer_choices: null - id: 3fa7947d-3e69-48c6-a1ad-780ab6a50e9f - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 285711ca-7f22-4e81-b56a-d909f3bd0467 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target + - version-de-en-source+target reference: '' - 5f25db3a-af08-4c3f-9aa0-6c88995298de: !Template + 2ca87d75-82bd-47bb-8f35-5330c2357bd0: !Template answer_choices: null - id: 5f25db3a-af08-4c3f-9aa0-6c88995298de - jinja: 'Translate German into English: {{translation["de"]}} + id: 2ca87d75-82bd-47bb-8f35-5330c2357bd0 + jinja: ' {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 6112b376-66fd-482b-a57f-7395c84f00ad: !Template + - translate_as_en-de-target + reference: '' + 3df26f11-69f5-4c35-bc41-2a2088cc3f5c: !Template answer_choices: null - id: 6112b376-66fd-482b-a57f-7395c84f00ad - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 3df26f11-69f5-4c35-bc41-2a2088cc3f5c + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target + - translate_as_en-de-source+target reference: '' - 6fa7b2bc-b98b-4d4e-af60-ff18619575fa: !Template + 56ba2774-5301-47b3-a727-c15ab3e57d98: !Template answer_choices: null - id: 6fa7b2bc-b98b-4d4e-af60-ff18619575fa - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: 56ba2774-5301-47b3-a727-c15ab3e57d98 + jinja: 'What is the English translation of : {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target - reference: '' - 7b5a9486-be04-4c3e-a623-5c7aea520727: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + 5a1aa7e3-4979-4e92-b9ae-5d24c9a7d8b7: !Template answer_choices: null - id: 7b5a9486-be04-4c3e-a623-5c7aea520727 - jinja: 'English: {{translation["en"]}} translates into German as: + id: 5a1aa7e3-4979-4e92-b9ae-5d24c9a7d8b7 + jinja: '{{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target - reference: '' - 88756e5a-f9e4-4beb-9c12-cfb8d0b0502a: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 6250fa75-6aa8-4942-a911-e858912a81fe: !Template answer_choices: null - id: 88756e5a-f9e4-4beb-9c12-cfb8d0b0502a - jinja: 'How do you say {{translation["en"]}} in German? + id: 6250fa75-6aa8-4942-a911-e858912a81fe + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - 892bc062-8275-4e7a-a4a6-40ec9ce917a5: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 6712c4ec-94f8-48e5-8766-65b99d24a2e0: !Template answer_choices: null - id: 892bc062-8275-4e7a-a4a6-40ec9ce917a5 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 6712c4ec-94f8-48e5-8766-65b99d24a2e0 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - version-de-en-target reference: '' - 89ea634c-a8c6-4a7a-8430-00dbcf70b94d: !Template + 71e66088-b150-4465-9912-01177de1a4ad: !Template answer_choices: null - id: 89ea634c-a8c6-4a7a-8430-00dbcf70b94d - jinja: 'Translate English into German: {{translation["en"]}} + id: 71e66088-b150-4465-9912-01177de1a4ad + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-en-target + reference: '' + 77063960-efd8-43a8-b9f4-171309efa924: !Template + answer_choices: null + id: 77063960-efd8-43a8-b9f4-171309efa924 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget + - a_good_translation-de-en-source+target reference: '' - 8a93706e-13dc-4e8c-a9bb-0a82567d650d: !Template + 78385c61-e86d-4838-a5df-47d2d41a44f6: !Template answer_choices: null - id: 8a93706e-13dc-4e8c-a9bb-0a82567d650d + id: 78385c61-e86d-4838-a5df-47d2d41a44f6 jinja: '{{translation["de"]}} translates into English as: ||| {{translation["en"]}}' @@ -198,25 +215,24 @@ templates: name: !!python/tuple - translate_as_de-en-target reference: '' - 94b1f824-97d2-4925-9f4f-e678b35e397d: !Template + 789c7529-7b54-4ca5-8d41-860f00ec6e12: !Template answer_choices: null - id: 94b1f824-97d2-4925-9f4f-e678b35e397d - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 789c7529-7b54-4ca5-8d41-860f00ec6e12 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target - reference: '' - af129188-cd5f-4fdb-bb3f-cbdd8c8c215e: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 871d391f-641c-4278-a6d2-42cebc56f3f7: !Template answer_choices: null - id: af129188-cd5f-4fdb-bb3f-cbdd8c8c215e - jinja: ' {{translation["en"]}} translates into German as: + id: 871d391f-641c-4278-a6d2-42cebc56f3f7 + jinja: '{{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -225,82 +241,81 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target - reference: '' - baf0d723-4dc0-4d53-a8d7-0123c524ae33: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 8f46d8fd-1d54-49d0-bce4-805cb3cd36bb: !Template answer_choices: null - id: baf0d723-4dc0-4d53-a8d7-0123c524ae33 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 8f46d8fd-1d54-49d0-bce4-805cb3cd36bb + jinja: 'What is the German translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target - reference: '' - bdd0c786-1d66-4213-a5b4-f5fb60ec118a: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + a8fa13e1-eb34-410b-9d50-c353a24c26b5: !Template answer_choices: null - id: bdd0c786-1d66-4213-a5b4-f5fb60ec118a - jinja: 'Translate into German: {{translation["en"]}} + id: a8fa13e1-eb34-410b-9d50-c353a24c26b5 + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - how_to_say-de-en-target reference: '' - d1f5d99e-c50e-4a05-961d-c6d20a0a689f: !Template + b86dadd8-d601-47d6-820c-3d65fb1db513: !Template answer_choices: null - id: d1f5d99e-c50e-4a05-961d-c6d20a0a689f - jinja: 'German: {{translation["de"]}} translates into English as: + id: b86dadd8-d601-47d6-820c-3d65fb1db513 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target - reference: '' - e2922fed-8ebb-4bf5-a538-7b406fab6739: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + c2860cd8-3358-4109-b07f-2b533c9224b8: !Template answer_choices: null - id: e2922fed-8ebb-4bf5-a538-7b406fab6739 - jinja: 'What is the German translation of: {{translation["en"]}} + id: c2860cd8-3358-4109-b07f-2b533c9224b8 + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: '' - e49a095f-1d19-4dc2-ab79-dd971c460bdb: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + d522ed8d-9b4b-44cc-b4c6-d966673eef3f: !Template answer_choices: null - id: e49a095f-1d19-4dc2-ab79-dd971c460bdb - jinja: 'How do you say {{translation["de"]}} in English? + id: d522ed8d-9b4b-44cc-b4c6-d966673eef3f + jinja: 'Translate into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - translate-de-en-target reference: '' - fc78b44f-a4fb-4d18-a9a8-cf3eff9f9b27: !Template + db0bcc05-3732-4931-9559-54c6ec56ec60: !Template answer_choices: null - id: fc78b44f-a4fb-4d18-a9a8-cf3eff9f9b27 + id: db0bcc05-3732-4931-9559-54c6ec56ec60 jinja: 'If the English version says: {{translation["en"]}}; then the German version should say: @@ -313,3 +328,44 @@ templates: name: !!python/tuple - version-en-de-source+target reference: '' + dc89baca-5066-4560-a98b-5d5fa5bbb0e3: !Template + answer_choices: null + id: dc89baca-5066-4560-a98b-5d5fa5bbb0e3 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + eb2dd5dd-1e4e-4235-827d-f63d1748015a: !Template + answer_choices: null + id: eb2dd5dd-1e4e-4235-827d-f63d1748015a + jinja: 'Translate English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-de-source+starget + reference: '' + f8099b32-c2dd-4b5d-bf8e-2ef9aeedf149: !Template + answer_choices: null + id: f8099b32-c2dd-4b5d-bf8e-2ef9aeedf149 + jinja: 'Translate German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf diff --git a/promptsource/templates/wmt18/et-en/templates.yaml b/promptsource/templates/wmt18/et-en/templates.yaml index 962d500c7..172f788ee 100644 --- a/promptsource/templates/wmt18/et-en/templates.yaml +++ b/promptsource/templates/wmt18/et-en/templates.yaml @@ -1,23 +1,37 @@ dataset: wmt18 subset: et-en templates: - 181992c7-17cc-4be2-9071-21e72add55bb: !Template + 09b66a7c-f6ef-448b-bcfd-0fa8e2ca7d05: !Template answer_choices: null - id: 181992c7-17cc-4be2-9071-21e72add55bb - jinja: 'What is the Estonian translation of: {{translation["en"]}} + id: 09b66a7c-f6ef-448b-bcfd-0fa8e2ca7d05 + jinja: '{{translation["et"]}} translates into English as: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-et-target + - translate_as_et-en-target reference: '' - 26187d14-713a-4ffc-b8c6-9cb6c28286b2: !Template + 12e126e6-87e4-4606-998f-274d75c0cdd4: !Template answer_choices: null - id: 26187d14-713a-4ffc-b8c6-9cb6c28286b2 + id: 12e126e6-87e4-4606-998f-274d75c0cdd4 + jinja: 'English: {{translation["en"]}} translates into Estonian as: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-et-source+target + reference: '' + 242d8884-3fab-4304-ae12-75f543979729: !Template + answer_choices: null + id: 242d8884-3fab-4304-ae12-75f543979729 jinja: 'If the original version says: {{translation["en"]}}; then the Estonian version should say: @@ -30,22 +44,23 @@ templates: name: !!python/tuple - version-en-et-target reference: '' - 39760f5a-2458-4d34-8d98-2c59799002ea: !Template + 3346e459-02ab-4a67-a0f3-0c4a97a5c7bc: !Template answer_choices: null - id: 39760f5a-2458-4d34-8d98-2c59799002ea - jinja: 'Given the following passage: {{translation["en"]}} , a good Estonian translation - is: ||| {{translation["et"]}}' + id: 3346e459-02ab-4a67-a0f3-0c4a97a5c7bc + jinja: '{{translation["en"]}} = Estonian: + + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-et-target - reference: '' - 3f96bd3d-73fa-4b75-9013-2ece6fdbd375: !Template + - xglm-en-et-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 367e49e9-b71a-4255-b213-a956c4e53dbc: !Template answer_choices: null - id: 3f96bd3d-73fa-4b75-9013-2ece6fdbd375 + id: 367e49e9-b71a-4255-b213-a956c4e53dbc jinja: 'What is the English translation of : {{translation["et"]}} ||| {{translation["en"]}}' @@ -56,11 +71,26 @@ templates: original_task: true name: !!python/tuple - gpt-3-et-en-target - reference: '' - 4bacbd39-c66d-4cb5-9b09-083276033257: !Template + reference: GPT-3 paper + 389fba3f-84b8-4d49-825a-af276178fef4: !Template answer_choices: null - id: 4bacbd39-c66d-4cb5-9b09-083276033257 - jinja: 'How do you say {{translation["en"]}} in Estonian? + id: 389fba3f-84b8-4d49-825a-af276178fef4 + jinja: 'What is the Estonian translation of: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-et-target + reference: GPT-3 paper + 4c2d841d-a772-4de2-9eec-d777cde53d0a: !Template + answer_choices: null + id: 4c2d841d-a772-4de2-9eec-d777cde53d0a + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Estonian translation is: ||| {{translation["et"]}}' metadata: !TemplateMetadata @@ -69,27 +99,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-et-target + - a_good_translation-en-et-source+target reference: '' - 5e548820-5c6d-41d3-bef1-6a004798efb0: !Template + 4d314556-2f70-4987-9537-0011e29f371f: !Template answer_choices: null - id: 5e548820-5c6d-41d3-bef1-6a004798efb0 - jinja: 'If the Estonian version says: {{translation["et"]}}; then the English - version should say: + id: 4d314556-2f70-4987-9537-0011e29f371f + jinja: ' {{translation["en"]}} translates into Estonian as: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-et-en-source+target + - translate_as_en-et-target reference: '' - 63f48c14-f8da-4c6f-b5aa-52680a0fc260: !Template + 4f053278-5fb1-4162-bf2b-3feb99630ef8: !Template answer_choices: null - id: 63f48c14-f8da-4c6f-b5aa-52680a0fc260 - jinja: 'How do you say {{translation["et"]}} in English? + id: 4f053278-5fb1-4162-bf2b-3feb99630ef8 + jinja: 'Estonian: {{translation["et"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -98,27 +127,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-et-en-target - reference: '' - 691c6136-2efb-40b5-9645-f2f69b935ed1: !Template + - xglm-et-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5287cf78-f2ab-41ec-a611-b516bf1dbe0f: !Template answer_choices: null - id: 691c6136-2efb-40b5-9645-f2f69b935ed1 - jinja: '{{translation["et"]}} translates into English as: + id: 5287cf78-f2ab-41ec-a611-b516bf1dbe0f + jinja: 'Translate into Estonian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_et-en-target + - translate-en-et-target reference: '' - 71b4588d-75f8-4df8-ade4-30f7399f0d41: !Template + 64ac7d67-94f2-4aa7-89b1-bc9247224fe1: !Template answer_choices: null - id: 71b4588d-75f8-4df8-ade4-30f7399f0d41 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Estonian translation is: + id: 64ac7d67-94f2-4aa7-89b1-bc9247224fe1 + jinja: 'How do you say {{translation["en"]}} in Estonian? ||| {{translation["et"]}}' metadata: !TemplateMetadata @@ -127,82 +155,83 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-et-source+target + - how_to_say-en-et-target reference: '' - 85f4a8ae-1fbf-4bc0-b4c7-e3859e181b9a: !Template + 65487caf-8d6f-4e36-965c-5bb8f75ab058: !Template answer_choices: null - id: 85f4a8ae-1fbf-4bc0-b4c7-e3859e181b9a - jinja: 'English: {{translation["en"]}} translates into Estonian as: + id: 65487caf-8d6f-4e36-965c-5bb8f75ab058 + jinja: 'What is the English translation of the Estonian sentence: {{translation["et"]}} - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-et-source+target - reference: '' - 87e69c86-0234-4d01-ae70-98a75a86f271: !Template + - gpt-3-et-en-source+target + reference: GPT-3 paper + 6591adaf-63ec-4b7f-ab45-f9647eeba93a: !Template answer_choices: null - id: 87e69c86-0234-4d01-ae70-98a75a86f271 - jinja: 'Given the following source text in Estonian: {{translation["et"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 6591adaf-63ec-4b7f-ab45-f9647eeba93a + jinja: 'Given the following passage: {{translation["et"]}} , a good English translation + is: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-et-en-source+target + - a_good_translation-et-en-target reference: '' - 94caadd3-0ef3-4ad5-912c-7074a7eb7896: !Template + 6831b64b-a425-49df-8f49-447f9dc0bb49: !Template answer_choices: null - id: 94caadd3-0ef3-4ad5-912c-7074a7eb7896 - jinja: 'If the original version says: {{translation["et"]}}; then the English - version should say: + id: 6831b64b-a425-49df-8f49-447f9dc0bb49 + jinja: 'English: {{translation["en"]}} = Estonian: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-et-en-target - reference: '' - 99758e66-c941-4daf-b474-bd3e0a88a940: !Template + - xglm-en-et-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 6ff1f5f8-4e88-4ed2-a8dd-9064eec947b4: !Template answer_choices: null - id: 99758e66-c941-4daf-b474-bd3e0a88a940 - jinja: 'If the English version says: {{translation["en"]}}; then the Estonian - version should say: + id: 6ff1f5f8-4e88-4ed2-a8dd-9064eec947b4 + jinja: 'How do you say {{translation["et"]}} in English? - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-et-source+target + - how_to_say-et-en-target reference: '' - 9b21eeec-25e7-47ba-af98-dc52bd9f3b38: !Template + 727c22ba-ee38-4d39-b7c2-5d3adf65bfb3: !Template answer_choices: null - id: 9b21eeec-25e7-47ba-af98-dc52bd9f3b38 - jinja: 'Translate English into Estonian: {{translation["en"]}} + id: 727c22ba-ee38-4d39-b7c2-5d3adf65bfb3 + jinja: 'If the original version says: {{translation["et"]}}; then the English + version should say: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-et-src+starget + - version-et-en-target reference: '' - 9f64040b-b37d-43f8-af82-b09d6225af41: !Template + 7d4c2271-06ba-443c-bdef-dfccc748c723: !Template answer_choices: null - id: 9f64040b-b37d-43f8-af82-b09d6225af41 + id: 7d4c2271-06ba-443c-bdef-dfccc748c723 jinja: 'What is the Estonian translation of the English sentence: {{translation["en"]}} ||| {{translation["et"]}}' @@ -212,39 +241,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-et-src+target - reference: '' - ae103af1-9d75-4e11-95e0-f14a1701c43a: !Template + - gpt-3-en-et-source+target + reference: GPT 3 paper + 81f0eb3e-7cc7-4bbc-bda1-cd2f2e688bc2: !Template answer_choices: null - id: ae103af1-9d75-4e11-95e0-f14a1701c43a - jinja: ' {{translation["en"]}} translates into Estonian as: + id: 81f0eb3e-7cc7-4bbc-bda1-cd2f2e688bc2 + jinja: '{{translation["et"]}} = English: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-et-target - reference: '' - b28fed6e-1099-43d0-af47-4329efa3ca10: !Template + - xglm-et-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 88c79995-83fa-4930-a3e1-676e352ea988: !Template answer_choices: null - id: b28fed6e-1099-43d0-af47-4329efa3ca10 - jinja: 'What is the English translation of the Estonian sentence: {{translation["et"]}} - - ||| {{translation["en"]}}' + id: 88c79995-83fa-4930-a3e1-676e352ea988 + jinja: 'Given the following passage: {{translation["en"]}} , a good Estonian translation + is: ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-et-en-src+target + - a_good_translation-en-et-target reference: '' - b41477f3-05c1-49d6-831a-1e8518f158e0: !Template + 8d1af6df-6641-4507-b611-70391e922292: !Template answer_choices: null - id: b41477f3-05c1-49d6-831a-1e8518f158e0 + id: 8d1af6df-6641-4507-b611-70391e922292 jinja: 'Translate Estonian into English: {{translation["et"]}} ||| {{translation["en"]}}' @@ -256,11 +284,24 @@ templates: name: !!python/tuple - translate-et-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - b6c2be75-43eb-4fc4-b279-c8503ca90bcc: !Template + 8e26f017-66be-4590-81f6-6e9ff144e1f4: !Template answer_choices: null - id: b6c2be75-43eb-4fc4-b279-c8503ca90bcc - jinja: 'Given the following passage: {{translation["et"]}} , a good English translation - is: + id: 8e26f017-66be-4590-81f6-6e9ff144e1f4 + jinja: 'Translate English into Estonian: {{translation["en"]}} + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-et-source+starget + reference: '' + 90ab0154-7745-4620-8b45-f655bee6a8e3: !Template + answer_choices: null + id: 90ab0154-7745-4620-8b45-f655bee6a8e3 + jinja: 'Estonian: {{translation["et"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -269,11 +310,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-et-en-target + - translate_as_et-en-source+target reference: '' - ba2b61ec-1a12-43e3-a055-b99917fc8ab0: !Template + a09ab74f-34b1-4a62-b8d1-6fbd29afb941: !Template answer_choices: null - id: ba2b61ec-1a12-43e3-a055-b99917fc8ab0 + id: a09ab74f-34b1-4a62-b8d1-6fbd29afb941 + jinja: 'If the English version says: {{translation["en"]}}; then the Estonian + version should say: + + ||| {{translation["et"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-et-source+target + reference: '' + c67279dc-b013-460f-b2dd-b046166faca2: !Template + answer_choices: null + id: c67279dc-b013-460f-b2dd-b046166faca2 jinja: 'Translate into English: {{translation["et"]}} ||| {{translation["en"]}}' @@ -285,31 +341,31 @@ templates: name: !!python/tuple - translate-et-en-target reference: '' - dce3295f-f22f-47b3-8d6c-464b5eb4dc82: !Template + e2cc5691-e61d-4bb6-8c61-d7fb67f3e88c: !Template answer_choices: null - id: dce3295f-f22f-47b3-8d6c-464b5eb4dc82 - jinja: 'Translate into Estonian: {{translation["en"]}} + id: e2cc5691-e61d-4bb6-8c61-d7fb67f3e88c + jinja: 'If the Estonian version says: {{translation["et"]}}; then the English + version should say: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-et-target + - version-et-en-source+target reference: '' - eaf56de6-ee11-472f-8fa1-c9aca0025e16: !Template + f263fce4-4f4a-46e3-983f-3d78dd73bbb9: !Template answer_choices: null - id: eaf56de6-ee11-472f-8fa1-c9aca0025e16 - jinja: 'Estonian: {{translation["et"]}} translates into English as: - - ||| {{translation["en"]}}' + id: f263fce4-4f4a-46e3-983f-3d78dd73bbb9 + jinja: 'Given the following source text in Estonian: {{translation["et"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_et-en-source+target + - a_good_translation-et-en-source+target reference: '' diff --git a/promptsource/templates/wmt18/fi-en/templates.yaml b/promptsource/templates/wmt18/fi-en/templates.yaml index 13b312dc4..3b46fe403 100644 --- a/promptsource/templates/wmt18/fi-en/templates.yaml +++ b/promptsource/templates/wmt18/fi-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt18 subset: fi-en templates: - 0a3bcad5-23df-419a-9fe6-4743aa9db2be: !Template + 024c1b4c-8c16-48bf-aa67-77b20793023b: !Template answer_choices: null - id: 0a3bcad5-23df-419a-9fe6-4743aa9db2be - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 024c1b4c-8c16-48bf-aa67-77b20793023b + jinja: 'What is the Finnish translation of: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -13,11 +13,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-src+starget - reference: '' - 210d9576-5a70-4c26-82fc-7b1bfbe01117: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 04b5ce95-a44f-4088-8236-f4620f34f680: !Template + answer_choices: null + id: 04b5ce95-a44f-4088-8236-f4620f34f680 + jinja: 'English: {{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 0b1028e7-82bd-4017-9cf7-9709338aec4e: !Template answer_choices: null - id: 210d9576-5a70-4c26-82fc-7b1bfbe01117 + id: 0b1028e7-82bd-4017-9cf7-9709338aec4e jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -28,69 +42,65 @@ templates: name: !!python/tuple - a_good_translation-en-fi-target reference: '' - 22b11d6c-e589-436b-b98a-e150d8d0efb8: !Template + 0fd8d2ba-6751-4f27-a52f-e17aaa390a88: !Template answer_choices: null - id: 22b11d6c-e589-436b-b98a-e150d8d0efb8 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: + id: 0fd8d2ba-6751-4f27-a52f-e17aaa390a88 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - how_to_say-en-fi-target reference: '' - 2350c901-a800-414a-b1dc-4b794aba207c: !Template + 19b94af0-cbb8-462c-8d3e-e0f9325931be: !Template answer_choices: null - id: 2350c901-a800-414a-b1dc-4b794aba207c - jinja: 'What is the Finnish translation of: {{translation["en"]}} - - ||| {{translation["fi"]}}' + id: 19b94af0-cbb8-462c-8d3e-e0f9325931be + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target + - a_good_translation-fi-en-source+target reference: '' - 30297da2-9cf8-4dc8-83c2-ad06db26b79c: !Template + 2d1ae41e-155c-4bc7-8795-0954d0d64136: !Template answer_choices: null - id: 30297da2-9cf8-4dc8-83c2-ad06db26b79c - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 2d1ae41e-155c-4bc7-8795-0954d0d64136 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - translate_as_fi-en-source+target reference: '' - 58fdc500-1ebf-47a0-85ba-4d556f80a168: !Template + 39b1cc28-71b1-4f0b-9f4e-49655855dfd2: !Template answer_choices: null - id: 58fdc500-1ebf-47a0-85ba-4d556f80a168 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: 39b1cc28-71b1-4f0b-9f4e-49655855dfd2 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target + - how_to_say-fi-en-target reference: '' - 6177d95f-6abe-402d-96e9-bf4a7b720fa8: !Template + 3ca1c531-f22a-4497-95fe-83c85d6bd0a9: !Template answer_choices: null - id: 6177d95f-6abe-402d-96e9-bf4a7b720fa8 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: 3ca1c531-f22a-4497-95fe-83c85d6bd0a9 + jinja: 'What is the English translation of : {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -99,25 +109,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-src+target - reference: '' - 8952e807-1e6a-4935-92d4-c6cd07610ddc: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + 494f82a7-2d20-4786-b1b6-08d985b8f1a5: !Template answer_choices: null - id: 8952e807-1e6a-4935-92d4-c6cd07610ddc - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 494f82a7-2d20-4786-b1b6-08d985b8f1a5 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - version-en-fi-target reference: '' - 979d391d-cfb8-410c-ac44-6911e716aea7: !Template + 4f12e1fb-7363-4357-a92a-8f1f8e5f5f9c: !Template answer_choices: null - id: 979d391d-cfb8-410c-ac44-6911e716aea7 + id: 4f12e1fb-7363-4357-a92a-8f1f8e5f5f9c jinja: 'Given the following source text in English: {{translation["en"]}} , a good Finnish translation is: @@ -130,11 +141,11 @@ templates: name: !!python/tuple - a_good_translation-en-fi-source+target reference: '' - 98a3ad8d-e40f-4804-8fea-895424a78cd1: !Template + 5386e7c0-40b2-4c0e-b85c-a2d4b7305937: !Template answer_choices: null - id: 98a3ad8d-e40f-4804-8fea-895424a78cd1 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 5386e7c0-40b2-4c0e-b85c-a2d4b7305937 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -143,12 +154,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - version-fi-en-target reference: '' - a73902fc-d455-48b2-9ab9-3924abfe005c: !Template + 6b8fc7a5-a109-4fcd-9e36-a52851cacd7d: !Template answer_choices: null - id: a73902fc-d455-48b2-9ab9-3924abfe005c - jinja: 'How do you say {{translation["fi"]}} in English? + id: 6b8fc7a5-a109-4fcd-9e36-a52851cacd7d + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -157,12 +169,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - a_good_translation-fi-en-target reference: '' - ad09a495-727a-4d09-b6bb-f2bf92d33c23: !Template + 6bef34c4-2e32-4f0c-b916-dc4c82626023: !Template answer_choices: null - id: ad09a495-727a-4d09-b6bb-f2bf92d33c23 - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 6bef34c4-2e32-4f0c-b916-dc4c82626023 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_fi-en-target + reference: '' + 74535955-e9bf-4450-9f1c-197a90c76f62: !Template + answer_choices: null + id: 74535955-e9bf-4450-9f1c-197a90c76f62 + jinja: 'Translate Finnish into English: {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -171,27 +197,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 7efa59e0-dd55-485e-b70c-8789f97cd7bc: !Template + answer_choices: null + id: 7efa59e0-dd55-485e-b70c-8789f97cd7bc + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target reference: '' - b1505a0a-fb50-47ca-815b-645eb93780e7: !Template + 896cd18d-d469-414e-959e-5925d12461ed: !Template answer_choices: null - id: b1505a0a-fb50-47ca-815b-645eb93780e7 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 896cd18d-d469-414e-959e-5925d12461ed + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - translate_as_en-fi-source+target reference: '' - b59c382b-457f-4d6a-9b63-7a8caf569ab7: !Template + 9455177d-c8d8-4106-bf52-239aef915196: !Template answer_choices: null - id: b59c382b-457f-4d6a-9b63-7a8caf569ab7 - jinja: '{{translation["fi"]}} translates into English as: + id: 9455177d-c8d8-4106-bf52-239aef915196 + jinja: 'Finnish: {{translation["fi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -200,38 +240,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target - reference: '' - be8c51f5-7cc7-49b2-9cee-8aecf2abe87b: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9488651a-e1ef-42d5-ae82-b5e0b2c8a98c: !Template answer_choices: null - id: be8c51f5-7cc7-49b2-9cee-8aecf2abe87b - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 9488651a-e1ef-42d5-ae82-b5e0b2c8a98c + jinja: '{{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target - reference: '' - cdf46e91-2228-4809-8c06-2b2c5b4fd700: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9c3f1a73-4b97-4e9a-b0c8-0657109974d9: !Template answer_choices: null - id: cdf46e91-2228-4809-8c06-2b2c5b4fd700 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 9c3f1a73-4b97-4e9a-b0c8-0657109974d9 + jinja: 'Translate into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e4103f0f-075b-481f-81b8-f3d5a3671c86: !Template + - translate-en-fi-target + reference: '' + b54d9dfe-bdd3-4c66-b698-d1fef339b856: !Template answer_choices: null - id: e4103f0f-075b-481f-81b8-f3d5a3671c86 + id: b54d9dfe-bdd3-4c66-b698-d1fef339b856 jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} ||| {{translation["fi"]}}' @@ -241,26 +282,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-src+target - reference: '' - ea3c2068-9385-4043-ac48-aef98ef9b26b: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + bd308723-64e4-4765-951f-6e10317a43fd: !Template answer_choices: null - id: ea3c2068-9385-4043-ac48-aef98ef9b26b - jinja: ' {{translation["en"]}} translates into Finnish as: + id: bd308723-64e4-4765-951f-6e10317a43fd + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - gpt-3-fi-en-source+target + reference: GPT-3 paper + c28c0e87-cba7-4374-bc79-c0bf36de7a23: !Template + answer_choices: null + id: c28c0e87-cba7-4374-bc79-c0bf36de7a23 + jinja: 'Translate into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-fi-en-target reference: '' - ee44f8c4-f689-4182-908a-a47679e31b07: !Template + cb5a91d5-c7ea-464e-8eb2-d5f029d176c2: !Template answer_choices: null - id: ee44f8c4-f689-4182-908a-a47679e31b07 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: cb5a91d5-c7ea-464e-8eb2-d5f029d176c2 + jinja: '{{translation["fi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -269,47 +324,48 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target - reference: '' - f0e9c69a-b0ea-46f0-9aa5-55cbbb518741: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + da6870c8-f33e-4809-a446-f72a0f376d40: !Template answer_choices: null - id: f0e9c69a-b0ea-46f0-9aa5-55cbbb518741 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: da6870c8-f33e-4809-a446-f72a0f376d40 + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - translate_as_en-fi-target reference: '' - f458a114-1e5e-43e0-819e-f91cc4a7f8a0: !Template + f715353e-4152-450b-8e6e-baf34a7b0672: !Template answer_choices: null - id: f458a114-1e5e-43e0-819e-f91cc4a7f8a0 - jinja: 'Translate into English: {{translation["fi"]}} + id: f715353e-4152-450b-8e6e-baf34a7b0672 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - version-en-fi-source+target reference: '' - fbc1a8bc-2b7c-4cdc-b79c-850dd9f1558b: !Template + fe8a576e-64f0-497a-a831-290cc1b693fd: !Template answer_choices: null - id: fbc1a8bc-2b7c-4cdc-b79c-850dd9f1558b - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: fe8a576e-64f0-497a-a831-290cc1b693fd + jinja: 'Translate English into Finnish: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - translate-en-fi-source+starget reference: '' diff --git a/promptsource/templates/wmt18/ru-en/templates.yaml b/promptsource/templates/wmt18/ru-en/templates.yaml index 80b1b092d..9cd03c8b0 100644 --- a/promptsource/templates/wmt18/ru-en/templates.yaml +++ b/promptsource/templates/wmt18/ru-en/templates.yaml @@ -1,9 +1,38 @@ dataset: wmt18 subset: ru-en templates: - 0c24a0e1-da87-452c-a600-776a539284e4: !Template + 01d74beb-afae-44b5-b29e-02041882f8ec: !Template answer_choices: null - id: 0c24a0e1-da87-452c-a600-776a539284e4 + id: 01d74beb-afae-44b5-b29e-02041882f8ec + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-ru-en-source+target + reference: '' + 06c3b21a-d5da-443d-950a-d826a360e209: !Template + answer_choices: null + id: 06c3b21a-d5da-443d-950a-d826a360e209 + jinja: 'English: {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-source+target + reference: '' + 0a6e3220-63d8-452c-8569-5fa60368ddc9: !Template + answer_choices: null + id: 0a6e3220-63d8-452c-8569-5fa60368ddc9 jinja: 'If the original version says: {{translation["ru"]}}; then the English version should say: @@ -16,9 +45,23 @@ templates: name: !!python/tuple - version-ru-en-target reference: '' - 0fd27cc6-b649-447c-a855-ce73ef3eb9c3: !Template + 14dc3a24-b45a-4e7c-b039-160162fd559d: !Template + answer_choices: null + id: 14dc3a24-b45a-4e7c-b039-160162fd559d + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_en-ru-target + reference: '' + 1e0006f8-30fa-49ea-bbbf-5e0c4cb1ef01: !Template answer_choices: null - id: 0fd27cc6-b649-447c-a855-ce73ef3eb9c3 + id: 1e0006f8-30fa-49ea-bbbf-5e0c4cb1ef01 jinja: 'Translate into English: {{translation["ru"]}} ||| {{translation["en"]}}' @@ -30,10 +73,10 @@ templates: name: !!python/tuple - translate-ru-en-target reference: '' - 1360a954-e86c-48cf-9345-7e658c307e15: !Template + 272d9f29-4228-4b3f-b996-1c9ca199f4ad: !Template answer_choices: null - id: 1360a954-e86c-48cf-9345-7e658c307e15 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 272d9f29-4228-4b3f-b996-1c9ca199f4ad + jinja: '{{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -42,27 +85,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target - reference: '' - 20534c85-0657-4966-bf6d-7d629f463cf4: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4224a9fa-fcf7-4613-a4d7-f14454ad367d: !Template answer_choices: null - id: 20534c85-0657-4966-bf6d-7d629f463cf4 - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 4224a9fa-fcf7-4613-a4d7-f14454ad367d + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - version-en-ru-source+target reference: '' - 25f15ee2-8ead-498a-9d32-586cb1207e2b: !Template + 4395a1cc-639b-481b-9ffc-2483b1fd0954: !Template answer_choices: null - id: 25f15ee2-8ead-498a-9d32-586cb1207e2b - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 4395a1cc-639b-481b-9ffc-2483b1fd0954 + jinja: 'English: {{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -71,82 +114,81 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target - reference: '' - 3cf75a76-473e-4297-88c5-11e0654d9b54: !Template + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 45cf6192-fee9-4e82-a4b4-51718ba2b9df: !Template answer_choices: null - id: 3cf75a76-473e-4297-88c5-11e0654d9b54 - jinja: '{{translation["ru"]}} translates into English as: + id: 45cf6192-fee9-4e82-a4b4-51718ba2b9df + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target - reference: '' - 589bae60-fec9-4bfe-b066-814b2a45e9c0: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 5909ecf7-85ea-42b8-b441-db560c81231f: !Template answer_choices: null - id: 589bae60-fec9-4bfe-b066-814b2a45e9c0 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 5909ecf7-85ea-42b8-b441-db560c81231f + jinja: 'How do you say {{translation["en"]}} in Russian? - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - how_to_say-en-ru-target reference: '' - 5fcca54f-b5c9-4083-8a5e-5ccc1b3d945d: !Template + 5b17adea-cfca-444f-8dec-aecc27168cad: !Template answer_choices: null - id: 5fcca54f-b5c9-4083-8a5e-5ccc1b3d945d - jinja: 'What is the English translation of : {{translation["ru"]}} - - ||| {{translation["en"]}}' + id: 5b17adea-cfca-444f-8dec-aecc27168cad + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target + - a_good_translation-en-ru-target reference: '' - 6262c46e-1f29-4e2d-a3fb-1d6d88107551: !Template + 619a1ee7-d2f3-46cc-9a88-fae912ab7d12: !Template answer_choices: null - id: 6262c46e-1f29-4e2d-a3fb-1d6d88107551 - jinja: 'What is the Russian translation of: {{translation["en"]}} - - ||| {{translation["ru"]}}' + id: 619a1ee7-d2f3-46cc-9a88-fae912ab7d12 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target + - a_good_translation-ru-en-source+target reference: '' - 669ee38b-d45c-46e8-8f06-5f7601f0d1ad: !Template + 634ce49c-d8b8-4715-9dd1-1c0b660ab7ba: !Template answer_choices: null - id: 669ee38b-d45c-46e8-8f06-5f7601f0d1ad - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 634ce49c-d8b8-4715-9dd1-1c0b660ab7ba + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 72bc2e97-139f-4e7c-a9fe-d46897341c91: !Template + - a_good_translation-ru-en-target + reference: '' + 6b990f93-754b-48bc-a1fd-afff9262fa2b: !Template answer_choices: null - id: 72bc2e97-139f-4e7c-a9fe-d46897341c91 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 6b990f93-754b-48bc-a1fd-afff9262fa2b + jinja: 'What is the English translation of : {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -155,40 +197,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target - reference: '' - 898213e4-f533-41fa-84fe-8af50182efee: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 6d0b2f15-f224-4c7f-8073-63de87134d6e: !Template answer_choices: null - id: 898213e4-f533-41fa-84fe-8af50182efee - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: 6d0b2f15-f224-4c7f-8073-63de87134d6e + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target - reference: '' - a0167177-6b60-4f88-ae61-759ae406fe37: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 95e1218e-e453-4c4f-8bd1-d0eae3c14d12: !Template answer_choices: null - id: a0167177-6b60-4f88-ae61-759ae406fe37 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 95e1218e-e453-4c4f-8bd1-d0eae3c14d12 + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - translate_as_ru-en-target reference: '' - a2058434-c434-4e31-a731-e38507384683: !Template + 9723af4f-23fd-451a-8c0e-082bc57dc139: !Template answer_choices: null - id: a2058434-c434-4e31-a731-e38507384683 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 9723af4f-23fd-451a-8c0e-082bc57dc139 + jinja: 'What is the Russian translation of: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -197,39 +239,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget - reference: '' - b19cb71d-b9d1-48e1-828a-b24a837be44b: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 9975a351-b686-4616-9e02-181d4388f283: !Template answer_choices: null - id: b19cb71d-b9d1-48e1-828a-b24a837be44b - jinja: ' {{translation["en"]}} translates into Russian as: + id: 9975a351-b686-4616-9e02-181d4388f283 + jinja: 'Translate into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate-en-ru-target reference: '' - b2300b3c-fa83-410d-b815-246c421ae934: !Template + 9ef1e600-e7e0-4ad4-8ba0-df4eb5abacf1: !Template answer_choices: null - id: b2300b3c-fa83-410d-b815-246c421ae934 - jinja: 'How do you say {{translation["ru"]}} in English? + id: 9ef1e600-e7e0-4ad4-8ba0-df4eb5abacf1 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - a_good_translation-en-ru-source+target reference: '' - b6d95e24-684f-4d12-858c-9b9bed9ddf38: !Template + a6e12617-5232-485d-a1a4-0a897dd8f826: !Template answer_choices: null - id: b6d95e24-684f-4d12-858c-9b9bed9ddf38 + id: a6e12617-5232-485d-a1a4-0a897dd8f826 jinja: 'If the original version says: {{translation["en"]}}; then the Russian version should say: @@ -242,38 +285,38 @@ templates: name: !!python/tuple - version-en-ru-target reference: '' - c71b2bfc-4184-41d1-af8d-ebe05c8d0b21: !Template + cd27bbef-a7ac-4db0-95a5-e2649877fcac: !Template answer_choices: null - id: c71b2bfc-4184-41d1-af8d-ebe05c8d0b21 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: cd27bbef-a7ac-4db0-95a5-e2649877fcac + jinja: 'Translate English into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - translate-en-ru-source+starget reference: '' - ce108686-f5b1-4727-9d4b-f6225fd06863: !Template + d4f5b5a0-c050-4eea-bfd4-3bb14fb6eb5b: !Template answer_choices: null - id: ce108686-f5b1-4727-9d4b-f6225fd06863 - jinja: 'Translate into Russian: {{translation["en"]}} + id: d4f5b5a0-c050-4eea-bfd4-3bb14fb6eb5b + jinja: 'Translate Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target - reference: '' - ceb4a956-96aa-485f-bb40-d003454de404: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e2baed91-95c3-40fa-b027-fc1d0344a932: !Template answer_choices: null - id: ceb4a956-96aa-485f-bb40-d003454de404 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: e2baed91-95c3-40fa-b027-fc1d0344a932 + jinja: 'How do you say {{translation["ru"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -282,34 +325,47 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - how_to_say-ru-en-target reference: '' - cecb7579-8a8d-435d-8734-76b3203aebb4: !Template + e5810f14-cd42-49dd-9542-51034230f4ff: !Template answer_choices: null - id: cecb7579-8a8d-435d-8734-76b3203aebb4 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: e5810f14-cd42-49dd-9542-51034230f4ff + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target - reference: '' - eff4c0f6-1b4e-41c8-8f77-0efaea477eab: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + e6bffe73-347f-4940-baa3-609f3a7a0528: !Template answer_choices: null - id: eff4c0f6-1b4e-41c8-8f77-0efaea477eab - jinja: 'How do you say {{translation["en"]}} in Russian? + id: e6bffe73-347f-4940-baa3-609f3a7a0528 + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + feff3a10-1258-48a7-91d2-d686417d6c24: !Template + answer_choices: null + id: feff3a10-1258-48a7-91d2-d686417d6c24 + jinja: 'Russian: {{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_ru-en-source+target reference: '' diff --git a/promptsource/templates/wmt18/tr-en/templates.yaml b/promptsource/templates/wmt18/tr-en/templates.yaml index 1af67ef71..991653512 100644 --- a/promptsource/templates/wmt18/tr-en/templates.yaml +++ b/promptsource/templates/wmt18/tr-en/templates.yaml @@ -1,65 +1,81 @@ dataset: wmt18 subset: tr-en templates: - 05958199-cfdc-43b3-bc2b-c85d8a676e7f: !Template + 016f1160-28ce-4177-ab95-f3c28d2a06c1: !Template answer_choices: null - id: 05958199-cfdc-43b3-bc2b-c85d8a676e7f - jinja: '{{translation["tr"]}} translates into English as: + id: 016f1160-28ce-4177-ab95-f3c28d2a06c1 + jinja: 'What is the English translation of : {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-target - reference: '' - 15124e31-d05e-409b-a5b1-0a1d2c2e3e21: !Template + - gpt-3-tr-en-target + reference: GPT-3 paper + 06300fbd-1570-404a-abfb-5ca26710a1d0: !Template answer_choices: null - id: 15124e31-d05e-409b-a5b1-0a1d2c2e3e21 - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: 06300fbd-1570-404a-abfb-5ca26710a1d0 + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-tr-en-source+target + reference: GPT-3 paper + 162fba92-4660-4293-8a92-8bdcbc90d739: !Template + answer_choices: null + id: 162fba92-4660-4293-8a92-8bdcbc90d739 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-src+starget + - a_good_translation-en-tr-source+target reference: '' - 1eca1c49-9140-4663-bd9d-d071fa0f4bef: !Template + 17064a42-05cd-497f-8387-64bf4083a6f2: !Template answer_choices: null - id: 1eca1c49-9140-4663-bd9d-d071fa0f4bef - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + id: 17064a42-05cd-497f-8387-64bf4083a6f2 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-src+target + - version-en-tr-target reference: '' - 282faae6-522d-4ccc-84a8-a8a4952495af: !Template + 22ef815c-051c-460a-96cb-f281eae08d06: !Template answer_choices: null - id: 282faae6-522d-4ccc-84a8-a8a4952495af - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: 22ef815c-051c-460a-96cb-f281eae08d06 + jinja: 'Translate English into Turkish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 3aec5f9b-b266-4ef6-bd0e-7f2d66de8b43: !Template + - translate-en-tr-source+starget + reference: '' + 274ee105-b03a-4fad-88da-fe6df67ecae6: !Template answer_choices: null - id: 3aec5f9b-b266-4ef6-bd0e-7f2d66de8b43 + id: 274ee105-b03a-4fad-88da-fe6df67ecae6 jinja: 'How do you say {{translation["tr"]}} in English? ||| {{translation["en"]}}' @@ -71,92 +87,93 @@ templates: name: !!python/tuple - how_to_say-tr-en-target reference: '' - 3c763b51-7749-432a-90aa-0d53e836bbde: !Template + 365d3c5a-abbf-486d-81d1-07974492fbd5: !Template answer_choices: null - id: 3c763b51-7749-432a-90aa-0d53e836bbde - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: 365d3c5a-abbf-486d-81d1-07974492fbd5 + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-src+target - reference: '' - 3e487ca8-786c-4ef3-8f9c-17946baf1b16: !Template + - gpt-3-en-tr-source+target + reference: GPT 3 paper + 3f36f729-8f09-4750-b935-0b68b3baf3b6: !Template answer_choices: null - id: 3e487ca8-786c-4ef3-8f9c-17946baf1b16 - jinja: 'What is the Turkish translation of: {{translation["en"]}} + id: 3f36f729-8f09-4750-b935-0b68b3baf3b6 + jinja: 'Translate Turkish into English: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target - reference: '' - 4af6842b-8267-4ae0-8679-20702a954f8c: !Template + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 4081db0d-365d-473d-bf0e-de8cb267a8d1: !Template answer_choices: null - id: 4af6842b-8267-4ae0-8679-20702a954f8c - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: 4081db0d-365d-473d-bf0e-de8cb267a8d1 + jinja: '{{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target - reference: '' - 4de63ac8-81e5-4b49-b08e-6bca4951e221: !Template + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 665c537e-e5f2-454e-ad32-fe81c5974905: !Template answer_choices: null - id: 4de63ac8-81e5-4b49-b08e-6bca4951e221 - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: 665c537e-e5f2-454e-ad32-fe81c5974905 + jinja: 'English: {{translation["en"]}} = Turkish: + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target - reference: '' - 4edee3cc-744d-4e41-8a23-f9afabb31217: !Template + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 6c8531d9-2003-4d45-8d67-59c67b71d7fd: !Template answer_choices: null - id: 4edee3cc-744d-4e41-8a23-f9afabb31217 - jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 6c8531d9-2003-4d45-8d67-59c67b71d7fd + jinja: ' {{translation["en"]}} translates into Turkish as: + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-source+target + - translate_as_en-tr-target reference: '' - 616d8492-6d4f-4543-bef3-932f73b7f61e: !Template + 6f78297a-47ff-4027-b3b0-d77d8663a1a9: !Template answer_choices: null - id: 616d8492-6d4f-4543-bef3-932f73b7f61e - jinja: 'Translate into English: {{translation["tr"]}} + id: 6f78297a-47ff-4027-b3b0-d77d8663a1a9 + jinja: 'Translate into Turkish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-target + - translate-en-tr-target reference: '' - 744e91e0-e30c-4f1b-a2d8-35a9347b0521: !Template + 71e3aef0-0719-4709-ae26-20740cb11360: !Template answer_choices: null - id: 744e91e0-e30c-4f1b-a2d8-35a9347b0521 + id: 71e3aef0-0719-4709-ae26-20740cb11360 jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation is: @@ -169,24 +186,36 @@ templates: name: !!python/tuple - a_good_translation-tr-en-target reference: '' - 75e28e26-bffe-490d-a708-900746621768: !Template + 801f0c40-fb74-4bf5-b21d-58855c8388ea: !Template answer_choices: null - id: 75e28e26-bffe-490d-a708-900746621768 - jinja: 'What is the English translation of : {{translation["tr"]}} - - ||| {{translation["en"]}}' + id: 801f0c40-fb74-4bf5-b21d-58855c8388ea + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target + - a_good_translation-tr-en-source+target reference: '' - 8945eb29-ad14-421e-8314-89aea8c36f7a: !Template + b08a3b75-f728-4255-98ed-2c29474d568c: !Template answer_choices: null - id: 8945eb29-ad14-421e-8314-89aea8c36f7a - jinja: 'How do you say {{translation["en"]}} in Turkish? + id: b08a3b75-f728-4255-98ed-2c29474d568c + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-tr-target + reference: '' + b283beaf-cdf2-4939-b91a-cd6c7fafbdfd: !Template + answer_choices: null + id: b283beaf-cdf2-4939-b91a-cd6c7fafbdfd + jinja: 'English: {{translation["en"]}} translates into Turkish as: ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -195,25 +224,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - translate_as_en-tr-source+target reference: '' - 8e138088-9e2b-461b-92d6-3b42c7b6e53c: !Template + c2bd250c-1c4e-47ea-99fa-7f5bd83ec2c7: !Template answer_choices: null - id: 8e138088-9e2b-461b-92d6-3b42c7b6e53c - jinja: ' {{translation["en"]}} translates into Turkish as: + id: c2bd250c-1c4e-47ea-99fa-7f5bd83ec2c7 + jinja: '{{translation["tr"]}} translates into English as: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target + - translate_as_tr-en-target reference: '' - 91af589e-45bb-438f-9001-776068e35f70: !Template + c8d30004-2ad5-45ac-9891-4b35fecb0193: !Template answer_choices: null - id: 91af589e-45bb-438f-9001-776068e35f70 + id: c8d30004-2ad5-45ac-9891-4b35fecb0193 jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version should say: @@ -226,24 +255,24 @@ templates: name: !!python/tuple - version-en-tr-source+target reference: '' - a11654bb-a98b-4439-85b0-3ec29a88d79a: !Template + c8eb00dd-3720-4cf0-a93b-742f087b9a8e: !Template answer_choices: null - id: a11654bb-a98b-4439-85b0-3ec29a88d79a - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: c8eb00dd-3720-4cf0-a93b-742f087b9a8e + jinja: 'Translate into English: {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target + - translate-tr-en-target reference: '' - b1313bde-5381-44c6-8b13-63775e846cf7: !Template + e2aa69d8-b11a-43df-900e-ad64e434ea9e: !Template answer_choices: null - id: b1313bde-5381-44c6-8b13-63775e846cf7 - jinja: 'English: {{translation["en"]}} translates into Turkish as: + id: e2aa69d8-b11a-43df-900e-ad64e434ea9e + jinja: 'How do you say {{translation["en"]}} in Turkish? ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -252,55 +281,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target + - how_to_say-en-tr-target reference: '' - b4224809-6324-4251-9c95-20b95aabe603: !Template + e487cf87-2e5f-44b2-a026-2978f4d27f8a: !Template answer_choices: null - id: b4224809-6324-4251-9c95-20b95aabe603 - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: e487cf87-2e5f-44b2-a026-2978f4d27f8a + jinja: 'What is the Turkish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target - reference: '' - c680f15d-0d5e-4652-9fba-0f13cbfbb833: !Template + - gpt-3-en-tr-target + reference: GPT-3 paper + e7d9f4d5-5687-4337-b0fb-fea27b4581e3: !Template answer_choices: null - id: c680f15d-0d5e-4652-9fba-0f13cbfbb833 - jinja: 'Translate into Turkish: {{translation["en"]}} + id: e7d9f4d5-5687-4337-b0fb-fea27b4581e3 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-target + - version-tr-en-target reference: '' - f50351f7-0d4d-485a-8ab3-caa2d0735b31: !Template + e9fed2be-f4d5-4437-a080-68db1c640f34: !Template answer_choices: null - id: f50351f7-0d4d-485a-8ab3-caa2d0735b31 - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: e9fed2be-f4d5-4437-a080-68db1c640f34 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target + - translate_as_tr-en-source+target reference: '' - fb7b4b9d-7346-4042-9d52-64f141ca8249: !Template + ed78f056-9059-4a95-821e-baef0831218a: !Template answer_choices: null - id: fb7b4b9d-7346-4042-9d52-64f141ca8249 + id: ed78f056-9059-4a95-821e-baef0831218a jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version should say: @@ -313,3 +341,31 @@ templates: name: !!python/tuple - version-tr-en-source+target reference: '' + f5942e98-b1db-4cd9-895d-3270aa9e15ce: !Template + answer_choices: null + id: f5942e98-b1db-4cd9-895d-3270aa9e15ce + jinja: '{{translation["en"]}} = Turkish: + + ||| {{translation["tr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-tr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f99a028e-61d1-4d1b-a75a-388d21cc253b: !Template + answer_choices: null + id: f99a028e-61d1-4d1b-a75a-388d21cc253b + jinja: 'Turkish: {{translation["tr"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt18/zh-en/templates.yaml b/promptsource/templates/wmt18/zh-en/templates.yaml index 5e5d13443..c2e07dc60 100644 --- a/promptsource/templates/wmt18/zh-en/templates.yaml +++ b/promptsource/templates/wmt18/zh-en/templates.yaml @@ -1,52 +1,52 @@ dataset: wmt18 subset: zh-en templates: - 0df95f1d-eac4-4a12-a03e-94836f4e5ecc: !Template + 0217111b-6e6f-4b66-a014-e904a87d9ba0: !Template answer_choices: null - id: 0df95f1d-eac4-4a12-a03e-94836f4e5ecc - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: 0217111b-6e6f-4b66-a014-e904a87d9ba0 + jinja: '{{translation["zh"]}} translates into English as: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target + - translate_as_zh-en-target reference: '' - 199b7514-847e-49e4-82c9-1ef4bf38ed26: !Template + 132b17ab-24be-4311-b78c-a9120a66ad30: !Template answer_choices: null - id: 199b7514-847e-49e4-82c9-1ef4bf38ed26 - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: 132b17ab-24be-4311-b78c-a9120a66ad30 + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target + - version-en-zh-target reference: '' - 19f5397e-de26-4234-a253-99f959d60bdf: !Template + 24b2947e-a498-4b3a-9b46-44a716915537: !Template answer_choices: null - id: 19f5397e-de26-4234-a253-99f959d60bdf - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Chinese translation is: + id: 24b2947e-a498-4b3a-9b46-44a716915537 + jinja: 'Translate into Chinese: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-source+target + - translate-en-zh-target reference: '' - 3f16f2a5-1b14-4fea-ac9d-b9ec7917f2fe: !Template + 2b306a49-9307-4dda-bf78-cca4a33e6ae6: !Template answer_choices: null - id: 3f16f2a5-1b14-4fea-ac9d-b9ec7917f2fe + id: 2b306a49-9307-4dda-bf78-cca4a33e6ae6 jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -57,38 +57,81 @@ templates: name: !!python/tuple - a_good_translation-zh-en-source+target reference: '' - 440f28fa-da51-4599-a014-3cdc6a3271d0: !Template + 3c810a32-b8e2-4e96-8b3a-ec7be3051b2c: !Template answer_choices: null - id: 440f28fa-da51-4599-a014-3cdc6a3271d0 - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: 3c810a32-b8e2-4e96-8b3a-ec7be3051b2c + jinja: 'English: {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target + - translate_as_en-zh-source+target reference: '' - 4858f6d2-e65e-4ffb-88da-2d299d935b56: !Template + 462b62aa-4d29-4fe6-b4e5-200c00d6ea4d: !Template answer_choices: null - id: 4858f6d2-e65e-4ffb-88da-2d299d935b56 - jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation - is: ||| {{translation["zh"]}}' + id: 462b62aa-4d29-4fe6-b4e5-200c00d6ea4d + jinja: 'How do you say {{translation["zh"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-target + - how_to_say-zh-en-target reference: '' - 49e8493b-084b-4994-a028-f423df05e95c: !Template + 49fb6c49-c090-416a-bb2f-de3ccb1f740f: !Template answer_choices: null - id: 49e8493b-084b-4994-a028-f423df05e95c - jinja: 'What is the English translation of : {{translation["zh"]}} + id: 49fb6c49-c090-416a-bb2f-de3ccb1f740f + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-zh-source+target + reference: GPT 3 paper + 534893e1-d429-42da-9ab7-a226456f5b99: !Template + answer_choices: null + id: 534893e1-d429-42da-9ab7-a226456f5b99 + jinja: '{{translation["zh"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5e73802f-3c78-444a-ba86-c4eaf811ce7e: !Template + answer_choices: null + id: 5e73802f-3c78-444a-ba86-c4eaf811ce7e + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-target + reference: '' + 6df05500-04c1-4bcd-b5eb-3e21e3500f77: !Template + answer_choices: null + id: 6df05500-04c1-4bcd-b5eb-3e21e3500f77 + jinja: 'Translate into English: {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -97,12 +140,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-target + - translate-zh-en-target reference: '' - 58cf0580-c83f-431d-b60c-d982285110a4: !Template + 723730ea-9efc-4fc8-9fb3-fdf6cb4d4f04: !Template answer_choices: null - id: 58cf0580-c83f-431d-b60c-d982285110a4 - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: 723730ea-9efc-4fc8-9fb3-fdf6cb4d4f04 + jinja: 'What is the Chinese translation of: {{translation["en"]}} ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -111,40 +154,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-src+target + - gpt-3-en-zh-target + reference: GPT-3 paper + 7e1d7244-5d3d-4a22-be30-022df5c4b04e: !Template + answer_choices: null + id: 7e1d7244-5d3d-4a22-be30-022df5c4b04e + jinja: 'How do you say {{translation["en"]}} in Chinese? + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-zh-target reference: '' - 66c75f62-572e-4fb3-84f2-0c8ee6b7285b: !Template + 8717c806-a408-4c2a-98b3-0caf4e3bf51f: !Template answer_choices: null - id: 66c75f62-572e-4fb3-84f2-0c8ee6b7285b - jinja: '{{translation["zh"]}} translates into English as: + id: 8717c806-a408-4c2a-98b3-0caf4e3bf51f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target + - a_good_translation-en-zh-source+target reference: '' - 79800cec-9256-41b5-855f-8a7075f4287e: !Template + 99e6cd94-60e3-4550-81a5-1905226f61a1: !Template answer_choices: null - id: 79800cec-9256-41b5-855f-8a7075f4287e - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: 99e6cd94-60e3-4550-81a5-1905226f61a1 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target + - translate_as_zh-en-source+target reference: '' - 7f159861-f50f-46b7-b806-9b77e5b6a13d: !Template + 9b059296-780f-4efc-af83-5a1f9de421c3: !Template answer_choices: null - id: 7f159861-f50f-46b7-b806-9b77e5b6a13d + id: 9b059296-780f-4efc-af83-5a1f9de421c3 jinja: 'Translate Chinese into English: {{translation["zh"]}} ||| {{translation["en"]}}' @@ -156,23 +213,24 @@ templates: name: !!python/tuple - translate-zh-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - 7ffd2f94-f31c-426a-b8f2-a6f98a82a3c8: !Template + 9ca6c947-df2c-40f4-91e7-4e8fa67c3028: !Template answer_choices: null - id: 7ffd2f94-f31c-426a-b8f2-a6f98a82a3c8 - jinja: 'How do you say {{translation["zh"]}} in English? + id: 9ca6c947-df2c-40f4-91e7-4e8fa67c3028 + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target + - version-en-zh-source+target reference: '' - 962010d1-7052-462e-a5f0-16bd18d8673f: !Template + aeb8aeb4-c0f2-40a9-96c6-3e33a7924e83: !Template answer_choices: null - id: 962010d1-7052-462e-a5f0-16bd18d8673f + id: aeb8aeb4-c0f2-40a9-96c6-3e33a7924e83 jinja: ' {{translation["en"]}} translates into Chinese as: ||| {{translation["zh"]}}' @@ -184,53 +242,52 @@ templates: name: !!python/tuple - translate_as_en-zh-target reference: '' - aba5c7bc-1bbe-4cbb-927c-d6184b05580a: !Template + bf32bab6-3e68-4ffb-b3cb-e2dc30d1260e: !Template answer_choices: null - id: aba5c7bc-1bbe-4cbb-927c-d6184b05580a - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese - version should say: - - ||| {{translation["zh"]}}' + id: bf32bab6-3e68-4ffb-b3cb-e2dc30d1260e + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target + - a_good_translation-en-zh-target reference: '' - aec5b3d3-5787-47cd-b2df-41a4b6a20219: !Template + cd1ad077-79b7-4799-b065-b899304a3f45: !Template answer_choices: null - id: aec5b3d3-5787-47cd-b2df-41a4b6a20219 - jinja: 'Translate into Chinese: {{translation["en"]}} + id: cd1ad077-79b7-4799-b065-b899304a3f45 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - version-zh-en-target reference: '' - c5689f05-ab0d-4e77-a712-4bb223dd29db: !Template + cdfdad6a-3f99-41af-bc5f-4f852c8c4df6: !Template answer_choices: null - id: c5689f05-ab0d-4e77-a712-4bb223dd29db - jinja: 'Translate English into Chinese: {{translation["en"]}} + id: cdfdad6a-3f99-41af-bc5f-4f852c8c4df6 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-src+starget - reference: '' - c82cb77c-0333-4dab-8472-f8864fd24148: !Template + - gpt-3-zh-en-source+target + reference: GPT-3 paper + df2baa97-9839-45db-bccb-2295cd10e0ac: !Template answer_choices: null - id: c82cb77c-0333-4dab-8472-f8864fd24148 - jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + id: df2baa97-9839-45db-bccb-2295cd10e0ac + jinja: 'What is the English translation of : {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -239,70 +296,69 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-src+target - reference: '' - cb635609-34c5-40c4-9464-9d66fcafb84c: !Template + - gpt-3-zh-en-target + reference: GPT-3 paper + e3f2165c-cae5-41ae-b947-0d90e42c9879: !Template answer_choices: null - id: cb635609-34c5-40c4-9464-9d66fcafb84c - jinja: 'Translate into English: {{translation["zh"]}} + id: e3f2165c-cae5-41ae-b947-0d90e42c9879 + jinja: 'Chinese: {{translation["zh"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-target - reference: '' - df192cf1-c074-4de6-ab60-6dd6c22ba4fc: !Template + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e472a8ba-7822-471a-b3ca-39db34e994fc: !Template answer_choices: null - id: df192cf1-c074-4de6-ab60-6dd6c22ba4fc - jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation - is: + id: e472a8ba-7822-471a-b3ca-39db34e994fc + jinja: 'English: {{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-target - reference: '' - e435476b-c028-4e7b-a7ff-bd1bf02a4c7b: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + e49a6349-658f-49af-a89a-b48ca1526c97: !Template answer_choices: null - id: e435476b-c028-4e7b-a7ff-bd1bf02a4c7b - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: e49a6349-658f-49af-a89a-b48ca1526c97 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target + - version-zh-en-source+target reference: '' - e6836792-5d2c-4a19-aefa-31f61a5e3d48: !Template + e9abd12f-8bdc-4092-a1ba-c747dea63876: !Template answer_choices: null - id: e6836792-5d2c-4a19-aefa-31f61a5e3d48 - jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version - should say: + id: e9abd12f-8bdc-4092-a1ba-c747dea63876 + jinja: '{{translation["en"]}} = Chinese: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-source+target - reference: '' - f736430b-f0c8-47bd-b261-30225b6ca1f3: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f66950cc-fdc2-44bd-bee3-3a5d230da47c: !Template answer_choices: null - id: f736430b-f0c8-47bd-b261-30225b6ca1f3 - jinja: 'What is the Chinese translation of: {{translation["en"]}} + id: f66950cc-fdc2-44bd-bee3-3a5d230da47c + jinja: 'Translate English into Chinese: {{translation["en"]}} ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-target + - translate-en-zh-source+starget reference: '' diff --git a/promptsource/templates/wmt19/de-en/templates.yaml b/promptsource/templates/wmt19/de-en/templates.yaml index 828333890..a35a55712 100644 --- a/promptsource/templates/wmt19/de-en/templates.yaml +++ b/promptsource/templates/wmt19/de-en/templates.yaml @@ -1,210 +1,208 @@ dataset: wmt19 subset: de-en templates: - 12c560a2-d578-4585-8b1c-c2d2dde494d0: !Template + 016f488b-e947-4082-becd-e5ce404ccf71: !Template answer_choices: null - id: 12c560a2-d578-4585-8b1c-c2d2dde494d0 - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: 016f488b-e947-4082-becd-e5ce404ccf71 + jinja: 'Translate into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - translate-de-en-target reference: '' - 16a88c9b-7408-415e-9acd-955725d46d11: !Template + 0702aabd-283c-455e-bb31-7798d6b6fabe: !Template answer_choices: null - id: 16a88c9b-7408-415e-9acd-955725d46d11 - jinja: 'German: {{translation["de"]}} translates into English as: + id: 0702aabd-283c-455e-bb31-7798d6b6fabe + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - version-de-en-target reference: '' - 1a0b5fa3-51e7-42d3-a3c3-e9f54bae86d0: !Template + 16cceacf-8525-4d04-8d83-4fa4cc9c6d1e: !Template answer_choices: null - id: 1a0b5fa3-51e7-42d3-a3c3-e9f54bae86d0 - jinja: 'Translate into German: {{translation["en"]}} + id: 16cceacf-8525-4d04-8d83-4fa4cc9c6d1e + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - how_to_say-de-en-target reference: '' - 22b45127-3245-4563-82fc-259ac56b1299: !Template + 1add00c0-9413-45ad-9a2b-17579f04abe6: !Template answer_choices: null - id: 22b45127-3245-4563-82fc-259ac56b1299 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 1add00c0-9413-45ad-9a2b-17579f04abe6 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target + - a_good_translation-en-de-source+target reference: '' - 39ffd410-aa87-4ea4-b9c1-f275fa819e40: !Template + 2c81808d-efc4-4d15-aa72-6ef02591c0a9: !Template answer_choices: null - id: 39ffd410-aa87-4ea4-b9c1-f275fa819e40 - jinja: ' {{translation["en"]}} translates into German as: + id: 2c81808d-efc4-4d15-aa72-6ef02591c0a9 + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target - reference: '' - 3a825c7b-b1c9-410b-9f2c-bd03e17cc8aa: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 323e9cac-0287-4cb3-b8f4-42b91a93e8f4: !Template answer_choices: null - id: 3a825c7b-b1c9-410b-9f2c-bd03e17cc8aa - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 323e9cac-0287-4cb3-b8f4-42b91a93e8f4 + jinja: 'Translate English into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - translate-en-de-source+starget reference: '' - 4503c426-c5f0-496a-a6c6-3bfb82a6d9b8: !Template + 42017cdc-da85-4c29-bf4e-692d20385c16: !Template answer_choices: null - id: 4503c426-c5f0-496a-a6c6-3bfb82a6d9b8 - jinja: 'What is the German translation of: {{translation["en"]}} + id: 42017cdc-da85-4c29-bf4e-692d20385c16 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target + - version-en-de-target reference: '' - 507d95a9-1866-4af3-ae4a-1c59a18859b6: !Template + 46272220-d718-4912-885d-b779ba22077a: !Template answer_choices: null - id: 507d95a9-1866-4af3-ae4a-1c59a18859b6 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: 46272220-d718-4912-885d-b779ba22077a + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - 57cda6d4-aa45-4410-b44e-fa4379a285fc: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 48ee7f3a-9a61-4106-9a95-7f9182a96ba1: !Template answer_choices: null - id: 57cda6d4-aa45-4410-b44e-fa4379a285fc - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 48ee7f3a-9a61-4106-9a95-7f9182a96ba1 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - how_to_say-en-de-target reference: '' - 7850c160-ac89-4d1c-b888-344f3a897a6c: !Template + 52177d5b-7b94-41e2-bcd9-143b5aab118e: !Template answer_choices: null - id: 7850c160-ac89-4d1c-b888-344f3a897a6c - jinja: 'Translate into English: {{translation["de"]}} + id: 52177d5b-7b94-41e2-bcd9-143b5aab118e + jinja: '{{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target - reference: '' - 86406e46-d972-4acb-95ba-416a0fd797ff: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5b09b443-0b71-4406-a4c4-7ca634e18aaf: !Template answer_choices: null - id: 86406e46-d972-4acb-95ba-416a0fd797ff - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 5b09b443-0b71-4406-a4c4-7ca634e18aaf + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - a_good_translation-de-en-source+target reference: '' - 8d5d8e4c-6a3e-4628-a665-58876376514b: !Template + 5d35fa9e-12ec-452b-8c0a-4b5b21c4b107: !Template answer_choices: null - id: 8d5d8e4c-6a3e-4628-a665-58876376514b - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 5d35fa9e-12ec-452b-8c0a-4b5b21c4b107 + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-src+target + - translate_as_de-en-target reference: '' - b969ff79-9878-4458-9bf0-1a65059c7e6d: !Template + 5ea19842-e310-4f29-bfe8-40555eb90ecb: !Template answer_choices: null - id: b969ff79-9878-4458-9bf0-1a65059c7e6d - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 5ea19842-e310-4f29-bfe8-40555eb90ecb + jinja: 'Translate German into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target - reference: '' - bfe56bc0-5579-4f9e-87b4-9b6f9e4dca40: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 74d50f3e-7ba9-46ad-a417-c155754e37f4: !Template answer_choices: null - id: bfe56bc0-5579-4f9e-87b4-9b6f9e4dca40 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: 74d50f3e-7ba9-46ad-a417-c155754e37f4 + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - translate_as_en-de-source+target reference: '' - c8d35e0b-e1fd-4d3e-b85b-41146355ea93: !Template + a1d31cb0-c10c-43d8-a272-2e9491b4b36d: !Template answer_choices: null - id: c8d35e0b-e1fd-4d3e-b85b-41146355ea93 - jinja: 'How do you say {{translation["de"]}} in English? + id: a1d31cb0-c10c-43d8-a272-2e9491b4b36d + jinja: 'German: {{translation["de"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -213,12 +211,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - translate_as_de-en-source+target reference: '' - cac26ba0-a319-4040-bda7-e3c40f354d66: !Template + aef2c049-27c2-4786-bc4a-37175592cdf9: !Template answer_choices: null - id: cac26ba0-a319-4040-bda7-e3c40f354d66 - jinja: '{{translation["de"]}} translates into English as: + id: aef2c049-27c2-4786-bc4a-37175592cdf9 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-de-target + reference: '' + b2aa1b51-21da-4fc1-80be-d19f12c378d7: !Template + answer_choices: null + id: b2aa1b51-21da-4fc1-80be-d19f12c378d7 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -227,12 +239,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - a_good_translation-de-en-target reference: '' - cc09b2e4-7903-427c-8cc4-23767ea2c0a7: !Template + b5c0e8be-bf2f-42d7-84c7-af453179188c: !Template answer_choices: null - id: cc09b2e4-7903-427c-8cc4-23767ea2c0a7 - jinja: 'English: {{translation["en"]}} translates into German as: + id: b5c0e8be-bf2f-42d7-84c7-af453179188c + jinja: ' {{translation["en"]}} translates into German as: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -241,68 +253,83 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - translate_as_en-de-target reference: '' - cdbb3f7f-fe3e-4498-9cad-26ab67155889: !Template + ba6e29d1-9143-4ac2-ba87-aa9043fa2b17: !Template answer_choices: null - id: cdbb3f7f-fe3e-4498-9cad-26ab67155889 - jinja: 'How do you say {{translation["en"]}} in German? + id: ba6e29d1-9143-4ac2-ba87-aa9043fa2b17 + jinja: '{{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - d950dcab-7731-4f89-8447-9566a1ffeb9f: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c456403b-179d-4653-8e26-bc9fd7b50c2e: !Template answer_choices: null - id: d950dcab-7731-4f89-8447-9566a1ffeb9f - jinja: 'What is the English translation of : {{translation["de"]}} + id: c456403b-179d-4653-8e26-bc9fd7b50c2e + jinja: 'Translate into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target + - translate-en-de-target reference: '' - dbc67f6b-a88d-4a4f-9977-ecee61cd03ec: !Template + c7f170d0-f8a9-4a02-a9b5-e8db22a6b76f: !Template answer_choices: null - id: dbc67f6b-a88d-4a4f-9977-ecee61cd03ec - jinja: 'Translate German into English: {{translation["de"]}} + id: c7f170d0-f8a9-4a02-a9b5-e8db22a6b76f + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - ea7609bc-c4b5-4c54-865d-35502fffeaa5: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + c843899a-1292-4129-84a1-7783c5d86184: !Template answer_choices: null - id: ea7609bc-c4b5-4c54-865d-35502fffeaa5 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: c843899a-1292-4129-84a1-7783c5d86184 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-src+target + - version-en-de-source+target reference: '' - f4dd1aef-4890-43a9-a860-ec1d3f915f3f: !Template + cad37129-c4f7-4fe7-9697-14ec07ddd78b: !Template answer_choices: null - id: f4dd1aef-4890-43a9-a860-ec1d3f915f3f - jinja: 'Translate English into German: {{translation["en"]}} + id: cad37129-c4f7-4fe7-9697-14ec07ddd78b + jinja: 'What is the German translation of: {{translation["en"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-de-target + reference: GPT-3 paper + d80689a8-f4da-4416-9939-3220c40c00aa: !Template + answer_choices: null + id: d80689a8-f4da-4416-9939-3220c40c00aa + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -311,5 +338,34 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-de-src+starget + - gpt-3-en-de-source+target + reference: GPT 3 paper + e0b2bece-f655-4279-997b-e893b94b04eb: !Template + answer_choices: null + id: e0b2bece-f655-4279-997b-e893b94b04eb + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-de-en-source+target reference: '' + f5accc29-205f-430b-a74e-54434b59b6f6: !Template + answer_choices: null + id: f5accc29-205f-430b-a74e-54434b59b6f6 + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-de-en-target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt19/fi-en/templates.yaml b/promptsource/templates/wmt19/fi-en/templates.yaml index 93f7e2a4e..34950b0d5 100644 --- a/promptsource/templates/wmt19/fi-en/templates.yaml +++ b/promptsource/templates/wmt19/fi-en/templates.yaml @@ -1,68 +1,94 @@ dataset: wmt19 subset: fi-en templates: - 1001cedb-97cb-47ad-8e7b-d50b03968dbc: !Template + 023d23ee-81ea-4257-bd52-61bf657498a8: !Template answer_choices: null - id: 1001cedb-97cb-47ad-8e7b-d50b03968dbc - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: 023d23ee-81ea-4257-bd52-61bf657498a8 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-src+target + - how_to_say-en-fi-target reference: '' - 10fa4a51-7d55-45a3-8cd7-7cba89f3b975: !Template + 0a833174-933a-447f-b5bc-84f2f9fb7966: !Template answer_choices: null - id: 10fa4a51-7d55-45a3-8cd7-7cba89f3b975 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + id: 0a833174-933a-447f-b5bc-84f2f9fb7966 + jinja: 'If the original version says: {{translation["fi"]}}; then the English version should say: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - version-fi-en-target reference: '' - 16d5b112-ec8c-4fe1-9352-472ec5681314: !Template + 17ecce9f-aa67-4735-abac-79727fd8ba87: !Template answer_choices: null - id: 16d5b112-ec8c-4fe1-9352-472ec5681314 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 17ecce9f-aa67-4735-abac-79727fd8ba87 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - a_good_translation-fi-en-target reference: '' - 1b5130e3-4630-41ad-9319-b3810fc02017: !Template + 22fb64db-ff31-4dbb-a88b-f986af30af07: !Template answer_choices: null - id: 1b5130e3-4630-41ad-9319-b3810fc02017 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 22fb64db-ff31-4dbb-a88b-f986af30af07 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-fi-target + reference: '' + 250690cf-c5c0-4b08-adea-dcebefd9182e: !Template + answer_choices: null + id: 250690cf-c5c0-4b08-adea-dcebefd9182e + jinja: 'Translate English into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - translate-en-fi-source+starget reference: '' - 2db9d5f4-456c-4dd0-8550-abe67de938de: !Template + 3d74225c-cd35-4501-bee2-54023e085209: !Template + answer_choices: null + id: 3d74225c-cd35-4501-bee2-54023e085209 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 51ab7868-5f60-49b6-91d5-42b01dc8ee78: !Template answer_choices: null - id: 2db9d5f4-456c-4dd0-8550-abe67de938de + id: 51ab7868-5f60-49b6-91d5-42b01dc8ee78 jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version should say: @@ -75,23 +101,38 @@ templates: name: !!python/tuple - version-en-fi-source+target reference: '' - 339f88b7-6f8a-4b9a-8a29-85dfcc6d6dc4: !Template + 53fce238-a0e0-43a0-8376-d5f652284024: !Template answer_choices: null - id: 339f88b7-6f8a-4b9a-8a29-85dfcc6d6dc4 - jinja: '{{translation["fi"]}} translates into English as: + id: 53fce238-a0e0-43a0-8376-d5f652284024 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 6a1c3d44-adb7-415b-bdcc-b36a10892432: !Template + answer_choices: null + id: 6a1c3d44-adb7-415b-bdcc-b36a10892432 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-fi-en-source+target reference: '' - 3e010328-0008-4452-b00d-1fee5f13f5db: !Template + 7497607e-de26-48bd-b679-453ac7eaa3f5: !Template answer_choices: null - id: 3e010328-0008-4452-b00d-1fee5f13f5db + id: 7497607e-de26-48bd-b679-453ac7eaa3f5 jinja: 'Translate into English: {{translation["fi"]}} ||| {{translation["en"]}}' @@ -103,107 +144,107 @@ templates: name: !!python/tuple - translate-fi-en-target reference: '' - 58713529-83aa-4b04-8baf-2f2c0df4de01: !Template + 7ab56181-5880-4ec0-a3a3-a080a3febbcc: !Template answer_choices: null - id: 58713529-83aa-4b04-8baf-2f2c0df4de01 - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 7ab56181-5880-4ec0-a3a3-a080a3febbcc + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target - reference: '' - 5d2e040b-4a63-4192-aa57-e4762208b9c4: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 7b10fd1d-d982-4fb6-a49e-31c2611a3e59: !Template answer_choices: null - id: 5d2e040b-4a63-4192-aa57-e4762208b9c4 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 7b10fd1d-d982-4fb6-a49e-31c2611a3e59 + jinja: 'Translate Finnish into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target - reference: '' - 61213da3-947e-452a-b693-bec694d4cdc0: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8476f748-b0a8-428e-b817-5178744d920e: !Template answer_choices: null - id: 61213da3-947e-452a-b693-bec694d4cdc0 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 8476f748-b0a8-428e-b817-5178744d920e + jinja: 'English: {{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target - reference: '' - 6568df70-fb26-481d-a87b-52840af9870b: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 88d5b90d-d1e4-4966-8598-74677f046168: !Template answer_choices: null - id: 6568df70-fb26-481d-a87b-52840af9870b - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 88d5b90d-d1e4-4966-8598-74677f046168 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 761c8d1e-bba4-4b1e-8289-a61a7002bfb7: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9218230a-6d3b-4a76-8580-7c5c01ee3ee1: !Template answer_choices: null - id: 761c8d1e-bba4-4b1e-8289-a61a7002bfb7 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: 9218230a-6d3b-4a76-8580-7c5c01ee3ee1 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - how_to_say-fi-en-target reference: '' - 782d11f2-5abf-43ef-aa4b-63bd537626fb: !Template + 92599ff2-6d98-487c-adae-9edba157a26f: !Template answer_choices: null - id: 782d11f2-5abf-43ef-aa4b-63bd537626fb - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 92599ff2-6d98-487c-adae-9edba157a26f + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target + - translate_as_en-fi-source+target reference: '' - 83535f54-e2c9-4048-81b8-44d815519021: !Template + 95674532-7c32-48fd-bc7b-4b097504a35d: !Template answer_choices: null - id: 83535f54-e2c9-4048-81b8-44d815519021 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 95674532-7c32-48fd-bc7b-4b097504a35d + jinja: ' {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target + - translate_as_en-fi-target reference: '' - 9b351621-e584-4f25-9aa1-2bd44296f166: !Template + 9bed380b-ae73-41a0-9e83-c5ee1ad04af2: !Template answer_choices: null - id: 9b351621-e584-4f25-9aa1-2bd44296f166 + id: 9bed380b-ae73-41a0-9e83-c5ee1ad04af2 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Finnish translation is: @@ -216,64 +257,79 @@ templates: name: !!python/tuple - a_good_translation-en-fi-source+target reference: '' - 9f87b91c-81be-4746-b5b7-404f8ce0cd8e: !Template + c1ec4708-85a5-4b30-be8f-618f983465e7: !Template answer_choices: null - id: 9f87b91c-81be-4746-b5b7-404f8ce0cd8e - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: c1ec4708-85a5-4b30-be8f-618f983465e7 + jinja: '{{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target + - translate_as_fi-en-target reference: '' - b8a9b65f-fb04-457d-b5ca-937fc746e7f0: !Template + c24a9fdb-86c6-48ad-946e-085688445f3a: !Template answer_choices: null - id: b8a9b65f-fb04-457d-b5ca-937fc746e7f0 - jinja: 'English: {{translation["en"]}} translates into Finnish as: - - ||| {{translation["fi"]}}' + id: c24a9fdb-86c6-48ad-946e-085688445f3a + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - a_good_translation-fi-en-source+target reference: '' - c8b36077-2743-4b30-b31f-609429b34dfe: !Template + c87d7a83-fb7d-461f-b43d-7ee5aecef31f: !Template answer_choices: null - id: c8b36077-2743-4b30-b31f-609429b34dfe - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: c87d7a83-fb7d-461f-b43d-7ee5aecef31f + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - version-en-fi-target reference: '' - caa1ab64-22c3-48e7-8f23-d36e89667f01: !Template + c8f66d73-f7d8-42d0-8207-332b73389354: !Template answer_choices: null - id: caa1ab64-22c3-48e7-8f23-d36e89667f01 - jinja: 'How do you say {{translation["fi"]}} in English? + id: c8f66d73-f7d8-42d0-8207-332b73389354 + jinja: 'Translate into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - translate-en-fi-target reference: '' - cab28bdd-009c-4a17-a9d1-388a932268d8: !Template + d515992b-4aee-4af8-b2b9-528d02b9915d: !Template + answer_choices: null + id: d515992b-4aee-4af8-b2b9-528d02b9915d + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["fi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-fi-source+target + reference: GPT 3 paper + de8d78ef-955c-4986-9223-ca3ff7a07aa6: !Template answer_choices: null - id: cab28bdd-009c-4a17-a9d1-388a932268d8 + id: de8d78ef-955c-4986-9223-ca3ff7a07aa6 jinja: 'Finnish: {{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' @@ -285,10 +341,10 @@ templates: name: !!python/tuple - translate_as_fi-en-source+target reference: '' - eb18840d-5589-4fd2-8680-64bf0cc14040: !Template + eeb1c1bd-de2e-4b54-883a-ab2cd9cd40ab: !Template answer_choices: null - id: eb18840d-5589-4fd2-8680-64bf0cc14040 - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: eeb1c1bd-de2e-4b54-883a-ab2cd9cd40ab + jinja: 'What is the Finnish translation of: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -297,19 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-src+starget - reference: '' - fd7d15e0-58d6-4aee-9623-4ceb47a2e3b5: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + fcb42e40-d756-401a-b021-2471b6342147: !Template answer_choices: null - id: fd7d15e0-58d6-4aee-9623-4ceb47a2e3b5 - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: fcb42e40-d756-401a-b021-2471b6342147 + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-src+target - reference: '' + - gpt-3-fi-en-target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt19/fr-de/templates.yaml b/promptsource/templates/wmt19/fr-de/templates.yaml index f207d4b8a..c6beb38c6 100644 --- a/promptsource/templates/wmt19/fr-de/templates.yaml +++ b/promptsource/templates/wmt19/fr-de/templates.yaml @@ -1,69 +1,67 @@ dataset: wmt19 subset: fr-de templates: - 06e5d46e-8710-43c8-bf44-c2303dd16cfd: !Template + 04f6d194-35a5-45f6-b2ba-edb54ad01956: !Template answer_choices: null - id: 06e5d46e-8710-43c8-bf44-c2303dd16cfd - jinja: 'If the original version says: {{translation["fr"]}}; then the German version - should say: + id: 04f6d194-35a5-45f6-b2ba-edb54ad01956 + jinja: '{{translation["fr"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fr-de-target + - translate_as_fr-de-target reference: '' - 0e4bda14-aa54-4dd4-8129-133f93125b80: !Template + 058c6082-3e5b-4c85-bf20-ba43f38947e4: !Template answer_choices: null - id: 0e4bda14-aa54-4dd4-8129-133f93125b80 - jinja: 'French: {{translation["fr"]}} translates into German as: + id: 058c6082-3e5b-4c85-bf20-ba43f38947e4 + jinja: 'Translate French into German: {{translation["fr"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-de-source+target - reference: '' - 109e862f-27ea-451d-b31f-653c018845fa: !Template + - translate-fr-de-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 09aa2188-5940-4e98-8758-ef1674c28299: !Template answer_choices: null - id: 109e862f-27ea-451d-b31f-653c018845fa - jinja: 'If the French version says: {{translation["fr"]}}; then the German version + id: 09aa2188-5940-4e98-8758-ef1674c28299 + jinja: 'If the original version says: {{translation["de"]}}; then the French version should say: - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fr-de-source+target + - version-de-fr-target reference: '' - 15ce04c0-8fca-4848-b211-bbc2911ab9a0: !Template + 0c05ce7b-e7c4-45fc-bcbd-890050290931: !Template answer_choices: null - id: 15ce04c0-8fca-4848-b211-bbc2911ab9a0 - jinja: 'German: {{translation["de"]}} translates into French as: + id: 0c05ce7b-e7c4-45fc-bcbd-890050290931 + jinja: 'What is the German translation of the French sentence: {{translation["fr"]}} - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-fr-source+target - reference: '' - 1e271797-2f0c-464e-bc96-b5015b4fcb06: !Template + - gpt-3-fr-de-source+target + reference: GPT-3 paper + 2605ab99-d28e-41c0-b5e4-6f5017f3929b: !Template answer_choices: null - id: 1e271797-2f0c-464e-bc96-b5015b4fcb06 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - French translation is: + id: 2605ab99-d28e-41c0-b5e4-6f5017f3929b + jinja: 'German: {{translation["de"]}} translates into French as: ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -72,67 +70,67 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-fr-source+target + - translate_as_de-fr-source+target reference: '' - 391bbba9-45e6-44e2-b706-612ddfd9a222: !Template + 282dab4a-90b8-4376-90e7-15ee3dac65cc: !Template answer_choices: null - id: 391bbba9-45e6-44e2-b706-612ddfd9a222 - jinja: 'If the German version says: {{translation["de"]}}; then the French version - should say: - - ||| {{translation["fr"]}}' + id: 282dab4a-90b8-4376-90e7-15ee3dac65cc + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + German translation is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-fr-source+target + - a_good_translation-fr-de-source+target reference: '' - 47192a46-77ae-45f2-8be6-aff4b06d479d: !Template + 2f0109ed-d052-438c-9475-1252eafb1597: !Template answer_choices: null - id: 47192a46-77ae-45f2-8be6-aff4b06d479d - jinja: '{{translation["fr"]}} translates into German as: + id: 2f0109ed-d052-438c-9475-1252eafb1597 + jinja: '{{translation["de"]}} = French: - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-de-target - reference: '' - 4bef3760-3594-489e-b9d7-94d9047d348c: !Template + - xglm-de-fr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 38776fc6-7e09-4250-923e-863d438833c0: !Template answer_choices: null - id: 4bef3760-3594-489e-b9d7-94d9047d348c - jinja: 'Given the following passage: {{translation["de"]}} , a good French translation - is: ||| {{translation["fr"]}}' + id: 38776fc6-7e09-4250-923e-863d438833c0 + jinja: 'What is the French translation of: {{translation["de"]}} + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-fr-target - reference: '' - 51476543-e538-482f-abe1-d334f432d879: !Template + - gpt-3-de-fr-target + reference: GPT-3 paper + 42a03567-18d2-41fe-ad01-2aae801c0eb8: !Template answer_choices: null - id: 51476543-e538-482f-abe1-d334f432d879 - jinja: 'What is the French translation of: {{translation["de"]}} + id: 42a03567-18d2-41fe-ad01-2aae801c0eb8 + jinja: 'If the German version says: {{translation["de"]}}; then the French version + should say: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-fr-target + - version-de-fr-source+target reference: '' - 5968a41f-823d-4c76-aead-c6b2642ed911: !Template + 430d0b3a-073f-4648-ba42-4705a87298ed: !Template answer_choices: null - id: 5968a41f-823d-4c76-aead-c6b2642ed911 + id: 430d0b3a-073f-4648-ba42-4705a87298ed jinja: 'How do you say {{translation["fr"]}} in German? ||| {{translation["de"]}}' @@ -144,51 +142,50 @@ templates: name: !!python/tuple - how_to_say-fr-de-target reference: '' - 5e7e4a8f-b7a9-4643-9eda-e0e4f264ac67: !Template + 51f9edfd-ce4c-481c-923a-88346fd5c58c: !Template answer_choices: null - id: 5e7e4a8f-b7a9-4643-9eda-e0e4f264ac67 - jinja: 'What is the German translation of : {{translation["fr"]}} + id: 51f9edfd-ce4c-481c-923a-88346fd5c58c + jinja: 'What is the French translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-de-target - reference: '' - 65fadf27-ecd0-4164-aa19-65d0124e32eb: !Template + - gpt-3-de-fr-source+target + reference: GPT 3 paper + 53d700f4-692a-462d-803a-458cb94d8315: !Template answer_choices: null - id: 65fadf27-ecd0-4164-aa19-65d0124e32eb - jinja: 'Translate German into French: {{translation["de"]}} - - ||| {{translation["fr"]}}' + id: 53d700f4-692a-462d-803a-458cb94d8315 + jinja: 'Given the following passage: {{translation["de"]}} , a good French translation + is: ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-fr-src+starget + - a_good_translation-de-fr-target reference: '' - 7ce086f0-f1cd-4ac6-9e62-c1debb631793: !Template + 7ba506d6-151b-4cf5-8b3d-659886f7ccd4: !Template answer_choices: null - id: 7ce086f0-f1cd-4ac6-9e62-c1debb631793 - jinja: 'Translate French into German: {{translation["fr"]}} + id: 7ba506d6-151b-4cf5-8b3d-659886f7ccd4 + jinja: 'How do you say {{translation["de"]}} in French? - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-de-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 7f129b9b-7f7e-4f33-a8ae-3d7141da1caa: !Template + - how_to_say-de-fr-target + reference: '' + 99aa7622-989e-4486-82e4-f3708886bcc1: !Template answer_choices: null - id: 7f129b9b-7f7e-4f33-a8ae-3d7141da1caa + id: 99aa7622-989e-4486-82e4-f3708886bcc1 jinja: 'Translate into German: {{translation["fr"]}} ||| {{translation["de"]}}' @@ -200,36 +197,38 @@ templates: name: !!python/tuple - translate-fr-de-target reference: '' - 8b31f232-6b09-4f80-b08d-990c00f571a1: !Template + ab18ee6c-7b28-4d95-91fc-74d788166709: !Template answer_choices: null - id: 8b31f232-6b09-4f80-b08d-990c00f571a1 - jinja: 'Given the following source text in French: {{translation["fr"]}} , a good - German translation is: ||| {{translation["de"]}}' + id: ab18ee6c-7b28-4d95-91fc-74d788166709 + jinja: 'German: {{translation["de"]}} = French: + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-de-source+target - reference: '' - a50c3409-850e-483d-aab3-db59cb467502: !Template + - xglm-de-fr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + adc7ebbc-8dd3-4c63-8341-a4787f9ab0f6: !Template answer_choices: null - id: a50c3409-850e-483d-aab3-db59cb467502 - jinja: 'What is the German translation of the French sentence: {{translation["fr"]}} + id: adc7ebbc-8dd3-4c63-8341-a4787f9ab0f6 + jinja: 'If the original version says: {{translation["fr"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-de-src+target + - version-fr-de-target reference: '' - c16b8dca-ca91-430b-828a-13ec0666570f: !Template + b2c178f3-e4ba-4b17-ac90-7145a3986dc6: !Template answer_choices: null - id: c16b8dca-ca91-430b-828a-13ec0666570f + id: b2c178f3-e4ba-4b17-ac90-7145a3986dc6 jinja: 'Given the following passage: {{translation["fr"]}} , a good German translation is: @@ -242,10 +241,10 @@ templates: name: !!python/tuple - a_good_translation-fr-de-target reference: '' - c5cf6a4a-3d88-497a-ab2a-c261a2acc4e0: !Template + c22ff3b0-df43-49b5-960b-51edc1b0e604: !Template answer_choices: null - id: c5cf6a4a-3d88-497a-ab2a-c261a2acc4e0 - jinja: 'How do you say {{translation["de"]}} in French? + id: c22ff3b0-df43-49b5-960b-51edc1b0e604 + jinja: ' {{translation["de"]}} translates into French as: ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -254,54 +253,82 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-fr-target + - translate_as_de-fr-target reference: '' - d5560280-7e1d-4425-a65e-9e3bef881399: !Template + c3d464e6-2c71-4deb-809b-330491a92438: !Template answer_choices: null - id: d5560280-7e1d-4425-a65e-9e3bef881399 - jinja: ' {{translation["de"]}} translates into French as: + id: c3d464e6-2c71-4deb-809b-330491a92438 + jinja: 'French: {{translation["fr"]}} = German: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-fr-target + - xglm-fr-de-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c3ed4191-deb0-4be0-9a06-479cced9a1c0: !Template + answer_choices: null + id: c3ed4191-deb0-4be0-9a06-479cced9a1c0 + jinja: '{{translation["fr"]}} = German: + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-fr-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c480958d-147f-4fa6-af10-d91551f1896e: !Template + answer_choices: null + id: c480958d-147f-4fa6-af10-d91551f1896e + jinja: 'Translate German into French: {{translation["de"]}} + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-fr-source+starget reference: '' - dd78b916-da02-4d4e-8e28-2a1fc12faf53: !Template + cf3caf39-8aa6-44ea-8586-7879d5fe5dec: !Template answer_choices: null - id: dd78b916-da02-4d4e-8e28-2a1fc12faf53 - jinja: 'If the original version says: {{translation["de"]}}; then the French version + id: cf3caf39-8aa6-44ea-8586-7879d5fe5dec + jinja: 'If the French version says: {{translation["fr"]}}; then the German version should say: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-fr-target + - version-fr-de-source+target reference: '' - e908e99f-8eb0-46c8-86d4-8d13b654b782: !Template + d5dd5907-05c5-4dec-bfa3-64036e63c1c9: !Template answer_choices: null - id: e908e99f-8eb0-46c8-86d4-8d13b654b782 - jinja: 'What is the French translation of the German sentence: {{translation["de"]}} + id: d5dd5907-05c5-4dec-bfa3-64036e63c1c9 + jinja: 'French: {{translation["fr"]}} translates into German as: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-fr-src+target + - translate_as_fr-de-source+target reference: '' - f37939c2-bc7b-43bc-90d4-2c368e6b3be8: !Template + f00b8dab-a49f-4a22-bd61-e8fe10221a98: !Template answer_choices: null - id: f37939c2-bc7b-43bc-90d4-2c368e6b3be8 + id: f00b8dab-a49f-4a22-bd61-e8fe10221a98 jinja: 'Translate into French: {{translation["de"]}} ||| {{translation["fr"]}}' @@ -313,3 +340,32 @@ templates: name: !!python/tuple - translate-de-fr-target reference: '' + f2bed3fa-546e-4d13-ab70-af3fa05ea384: !Template + answer_choices: null + id: f2bed3fa-546e-4d13-ab70-af3fa05ea384 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + French translation is: + + ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-fr-source+target + reference: '' + fedb1953-e3cf-4c48-aa78-f10cdef832fc: !Template + answer_choices: null + id: fedb1953-e3cf-4c48-aa78-f10cdef832fc + jinja: 'What is the German translation of : {{translation["fr"]}} + + ||| {{translation["de"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-fr-de-target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt19/gu-en/templates.yaml b/promptsource/templates/wmt19/gu-en/templates.yaml index e3b2f2eb9..6692ad3ec 100644 --- a/promptsource/templates/wmt19/gu-en/templates.yaml +++ b/promptsource/templates/wmt19/gu-en/templates.yaml @@ -1,9 +1,52 @@ dataset: wmt19 subset: gu-en templates: - 024097fc-b7b4-487b-bd65-427f69fc2f9f: !Template + 3356cbf5-bab8-48b8-8930-aa27a0e428e4: !Template answer_choices: null - id: 024097fc-b7b4-487b-bd65-427f69fc2f9f + id: 3356cbf5-bab8-48b8-8930-aa27a0e428e4 + jinja: 'Translate into Gurajati: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-gu-target + reference: '' + 39cb6dd4-1b9e-45bc-aeb5-b43066a84e0f: !Template + answer_choices: null + id: 39cb6dd4-1b9e-45bc-aeb5-b43066a84e0f + jinja: 'Given the following passage: {{translation["gu"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-gu-en-target + reference: '' + 41a4a411-56dc-4e2f-b616-6f41a2e7cd3b: !Template + answer_choices: null + id: 41a4a411-56dc-4e2f-b616-6f41a2e7cd3b + jinja: 'What is the Gurajati translation of: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-en-gu-target + reference: GPT-3 paper + 4f119283-2149-4fc5-a0e2-947a30ca0b6e: !Template + answer_choices: null + id: 4f119283-2149-4fc5-a0e2-947a30ca0b6e jinja: 'If the English version says: {{translation["en"]}}; then the Gurajati version should say: @@ -16,10 +59,10 @@ templates: name: !!python/tuple - version-en-gu-source+target reference: '' - 1508501b-a090-4d29-aa7e-faa8fef35aa0: !Template + 5171b947-ab61-4cbf-8708-b6525bef6444: !Template answer_choices: null - id: 1508501b-a090-4d29-aa7e-faa8fef35aa0 - jinja: 'Translate Gurajati into English: {{translation["gu"]}} + id: 5171b947-ab61-4cbf-8708-b6525bef6444 + jinja: 'Translate into English: {{translation["gu"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -28,40 +71,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-gu-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1a362864-c9a8-47f2-8e94-b1b78b7f02df: !Template + - translate-gu-en-target + reference: '' + 6441d5dd-b04b-4060-9644-4e57e6f1cd90: !Template answer_choices: null - id: 1a362864-c9a8-47f2-8e94-b1b78b7f02df - jinja: 'If the Gurajati version says: {{translation["gu"]}}; then the English + id: 6441d5dd-b04b-4060-9644-4e57e6f1cd90 + jinja: 'If the original version says: {{translation["en"]}}; then the Gurajati version should say: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-gu-en-source+target + - version-en-gu-target reference: '' - 23ce908d-804c-42d0-8ba1-abf39d614501: !Template + 684c70ae-f44f-4fb3-b387-67c6e2f46506: !Template answer_choices: null - id: 23ce908d-804c-42d0-8ba1-abf39d614501 - jinja: 'How do you say {{translation["gu"]}} in English? - - ||| {{translation["en"]}}' + id: 684c70ae-f44f-4fb3-b387-67c6e2f46506 + jinja: 'Given the following source text in Gurajati: {{translation["gu"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-gu-en-target + - a_good_translation-gu-en-source+target reference: '' - 2d883e74-d1de-4fb5-864a-0826b93aeffa: !Template + 697fba8f-c487-4dbe-a39c-4c3dee1be014: !Template answer_choices: null - id: 2d883e74-d1de-4fb5-864a-0826b93aeffa + id: 697fba8f-c487-4dbe-a39c-4c3dee1be014 jinja: 'What is the Gurajati translation of the English sentence: {{translation["en"]}} ||| {{translation["gu"]}}' @@ -71,26 +113,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-gu-src+target - reference: '' - 3cc30841-6b2d-4fa7-9c0b-a1e9c276a3b0: !Template + - gpt-3-en-gu-source+target + reference: GPT 3 paper + 710d1d8e-d27a-4360-b77c-557e7ef6f147: !Template answer_choices: null - id: 3cc30841-6b2d-4fa7-9c0b-a1e9c276a3b0 - jinja: '{{translation["gu"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 710d1d8e-d27a-4360-b77c-557e7ef6f147 + jinja: 'Given the following passage: {{translation["en"]}} , a good Gurajati translation + is: ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_gu-en-target + - a_good_translation-en-gu-target reference: '' - 42b4d3e8-5dba-4b0d-8c5e-af24d69359a6: !Template + 7837b4a7-3de5-4e15-ac1a-35c2e7850a0e: !Template answer_choices: null - id: 42b4d3e8-5dba-4b0d-8c5e-af24d69359a6 - jinja: ' {{translation["en"]}} translates into Gurajati as: + id: 7837b4a7-3de5-4e15-ac1a-35c2e7850a0e + jinja: 'English: {{translation["en"]}} translates into Gurajati as: ||| {{translation["gu"]}}' metadata: !TemplateMetadata @@ -99,55 +140,56 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-gu-target + - translate_as_en-gu-source+target reference: '' - 493595fb-0f0e-4821-a726-e1a764e8138f: !Template + 7edc2e1e-e96b-4012-b90a-9e37d075173c: !Template answer_choices: null - id: 493595fb-0f0e-4821-a726-e1a764e8138f - jinja: 'Translate English into Gurajati: {{translation["en"]}} + id: 7edc2e1e-e96b-4012-b90a-9e37d075173c + jinja: 'Gurajati: {{translation["gu"]}} translates into English as: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-gu-src+starget + - translate_as_gu-en-source+target reference: '' - 4a62cf8b-cff0-433d-b69c-afbfe78254e2: !Template + 849b7ad4-3be7-415d-997e-a9013b2b4447: !Template answer_choices: null - id: 4a62cf8b-cff0-433d-b69c-afbfe78254e2 - jinja: 'Given the following passage: {{translation["gu"]}} , a good English translation - is: + id: 849b7ad4-3be7-415d-997e-a9013b2b4447 + jinja: 'If the Gurajati version says: {{translation["gu"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-gu-en-target + - version-gu-en-source+target reference: '' - 4d82a1d8-ceb6-4258-a55b-b7473a3f294b: !Template + 8cf8e194-abb9-43e6-a1e1-e8d3b9e5423f: !Template answer_choices: null - id: 4d82a1d8-ceb6-4258-a55b-b7473a3f294b - jinja: 'What is the Gurajati translation of: {{translation["en"]}} + id: 8cf8e194-abb9-43e6-a1e1-e8d3b9e5423f + jinja: 'If the original version says: {{translation["gu"]}}; then the English + version should say: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-gu-target + - version-gu-en-target reference: '' - 5910df5c-7738-4706-880c-a2f8e0e66f03: !Template + 97ceebb5-89a8-47e7-b5f8-49e070f3af8b: !Template answer_choices: null - id: 5910df5c-7738-4706-880c-a2f8e0e66f03 - jinja: 'Gurajati: {{translation["gu"]}} translates into English as: + id: 97ceebb5-89a8-47e7-b5f8-49e070f3af8b + jinja: 'How do you say {{translation["gu"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -156,40 +198,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_gu-en-source+target + - how_to_say-gu-en-target reference: '' - 6531eac2-5d58-48f2-963b-d617a084d57b: !Template + 9a5136ee-169b-45d8-ad8b-67765661da30: !Template answer_choices: null - id: 6531eac2-5d58-48f2-963b-d617a084d57b - jinja: 'English: {{translation["en"]}} translates into Gurajati as: + id: 9a5136ee-169b-45d8-ad8b-67765661da30 + jinja: 'Gurajati: {{translation["gu"]}} = English: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-gu-source+target - reference: '' - 7200ee57-175b-4a62-b408-0ad05efb8a4f: !Template + - xglm-gu-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b973b4b8-2dd5-407f-86db-947f40168c67: !Template answer_choices: null - id: 7200ee57-175b-4a62-b408-0ad05efb8a4f - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Gurajati translation is: + id: b973b4b8-2dd5-407f-86db-947f40168c67 + jinja: '{{translation["gu"]}} translates into English as: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-gu-source+target + - translate_as_gu-en-target reference: '' - 78d0b841-b9dc-49fa-8546-ef5736490e69: !Template + d543de51-a465-4389-b155-400ac558579a: !Template answer_choices: null - id: 78d0b841-b9dc-49fa-8546-ef5736490e69 + id: d543de51-a465-4389-b155-400ac558579a jinja: 'How do you say {{translation["en"]}} in Gurajati? ||| {{translation["gu"]}}' @@ -201,108 +242,109 @@ templates: name: !!python/tuple - how_to_say-en-gu-target reference: '' - a59fa17f-6710-4a73-8e98-7926b3b1147d: !Template + da75185f-a87d-400f-bfde-5fc6f668a91d: !Template answer_choices: null - id: a59fa17f-6710-4a73-8e98-7926b3b1147d - jinja: 'If the original version says: {{translation["en"]}}; then the Gurajati - version should say: + id: da75185f-a87d-400f-bfde-5fc6f668a91d + jinja: ' {{translation["en"]}} translates into Gurajati as: - ||| {{translation["gu"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-gu-target + - translate_as_en-gu-target reference: '' - b6f836db-a51e-4a9b-a5eb-153248a88df9: !Template + e4ff9670-a294-4b3f-bc32-f9779669ee16: !Template answer_choices: null - id: b6f836db-a51e-4a9b-a5eb-153248a88df9 - jinja: 'Given the following source text in Gurajati: {{translation["gu"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: e4ff9670-a294-4b3f-bc32-f9779669ee16 + jinja: 'Translate Gurajati into English: {{translation["gu"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-gu-en-source+target - reference: '' - b8f138f2-127e-4107-aa19-51ccf59fcebf: !Template + - translate-gu-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e64f9072-af1a-4f09-aad7-f55280518f8b: !Template answer_choices: null - id: b8f138f2-127e-4107-aa19-51ccf59fcebf - jinja: 'What is the English translation of : {{translation["gu"]}} + id: e64f9072-af1a-4f09-aad7-f55280518f8b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Gurajati translation is: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-gu-en-target + - a_good_translation-en-gu-source+target reference: '' - bc0b95d1-dbba-42df-8c68-bd9219fd9ad6: !Template + e86e178f-db53-491a-aac3-c2b80a0c6f27: !Template answer_choices: null - id: bc0b95d1-dbba-42df-8c68-bd9219fd9ad6 - jinja: 'What is the English translation of the Gurajati sentence: {{translation["gu"]}} + id: e86e178f-db53-491a-aac3-c2b80a0c6f27 + jinja: 'English: {{translation["en"]}} = Gurajati: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-gu-en-src+target - reference: '' - c5bb6887-f5be-426f-a275-a456a5ffe4dd: !Template + - xglm-en-gu-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + ec1f81e9-8b1d-44de-a503-68b67f9e1ffa: !Template answer_choices: null - id: c5bb6887-f5be-426f-a275-a456a5ffe4dd - jinja: 'Translate into Gurajati: {{translation["en"]}} + id: ec1f81e9-8b1d-44de-a503-68b67f9e1ffa + jinja: '{{translation["gu"]}} = English: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-gu-target - reference: '' - df05bb02-ef33-4e4e-b16e-0ec9a2cb067b: !Template + - xglm-gu-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f275a19c-3927-4073-8abd-d39d97f3f70f: !Template answer_choices: null - id: df05bb02-ef33-4e4e-b16e-0ec9a2cb067b - jinja: 'If the original version says: {{translation["gu"]}}; then the English - version should say: + id: f275a19c-3927-4073-8abd-d39d97f3f70f + jinja: '{{translation["en"]}} = Gurajati: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-gu-en-target - reference: '' - ed90082f-a447-4830-8842-5d8ab6d59f5c: !Template + - xglm-en-gu-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f43e4d43-b409-48ef-8e16-1c21a05f8369: !Template answer_choices: null - id: ed90082f-a447-4830-8842-5d8ab6d59f5c - jinja: 'Given the following passage: {{translation["en"]}} , a good Gurajati translation - is: ||| {{translation["gu"]}}' + id: f43e4d43-b409-48ef-8e16-1c21a05f8369 + jinja: 'What is the English translation of : {{translation["gu"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-gu-target - reference: '' - fe5a2406-1643-421f-b84d-bee3efe549b5: !Template + - gpt-3-gu-en-target + reference: GPT-3 paper + f56533a5-e686-4ff1-8821-08d3482af548: !Template answer_choices: null - id: fe5a2406-1643-421f-b84d-bee3efe549b5 - jinja: 'Translate into English: {{translation["gu"]}} + id: f56533a5-e686-4ff1-8821-08d3482af548 + jinja: 'What is the English translation of the Gurajati sentence: {{translation["gu"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-gu-en-target + - gpt-3-gu-en-source+target + reference: GPT-3 paper + f6d40b63-a53d-4f09-9d57-22e41a862baa: !Template + answer_choices: null + id: f6d40b63-a53d-4f09-9d57-22e41a862baa + jinja: 'Translate English into Gurajati: {{translation["en"]}} + + ||| {{translation["gu"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-en-gu-source+starget reference: '' diff --git a/promptsource/templates/wmt19/kk-en/templates.yaml b/promptsource/templates/wmt19/kk-en/templates.yaml index 53d19b26c..9c7076d6a 100644 --- a/promptsource/templates/wmt19/kk-en/templates.yaml +++ b/promptsource/templates/wmt19/kk-en/templates.yaml @@ -1,182 +1,211 @@ dataset: wmt19 subset: kk-en templates: - 157c74e2-4eaf-492a-b3d5-b7b55df8a8f0: !Template + 19bc7d44-4965-40b8-9198-653b1766871c: !Template answer_choices: null - id: 157c74e2-4eaf-492a-b3d5-b7b55df8a8f0 - jinja: 'How do you say {{translation["en"]}} in Kazakh? + id: 19bc7d44-4965-40b8-9198-653b1766871c + jinja: '{{translation["kk"]}} translates into English as: - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-kk-target + - translate_as_kk-en-target reference: '' - 175347ec-2186-4a10-8b83-f88a604c87e2: !Template + 1d10e5f9-ae18-42c1-a752-d3d4543e3bc5: !Template answer_choices: null - id: 175347ec-2186-4a10-8b83-f88a604c87e2 - jinja: 'If the original version says: {{translation["kk"]}}; then the English - version should say: + id: 1d10e5f9-ae18-42c1-a752-d3d4543e3bc5 + jinja: 'If the English version says: {{translation["en"]}}; then the Kazakh version + should say: - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-kk-en-target + - version-en-kk-source+target reference: '' - 1d967f47-7d17-4755-9399-b436f9dd3267: !Template + 1d87080c-c1b6-49a7-aced-a6aa96da7f7c: !Template answer_choices: null - id: 1d967f47-7d17-4755-9399-b436f9dd3267 - jinja: 'If the original version says: {{translation["en"]}}; then the Kazakh version - should say: + id: 1d87080c-c1b6-49a7-aced-a6aa96da7f7c + jinja: 'Kazakh: {{translation["kk"]}} = English: - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-kk-target - reference: '' - 271a87d3-43cb-4962-9446-ee5df99912dc: !Template + - xglm-kk-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 1f5d3cf4-76cf-4629-875e-e9730ef17594: !Template answer_choices: null - id: 271a87d3-43cb-4962-9446-ee5df99912dc - jinja: 'What is the Kazakh translation of the English sentence: {{translation["en"]}} + id: 1f5d3cf4-76cf-4629-875e-e9730ef17594 + jinja: 'English: {{translation["en"]}} translates into Kazakh as: - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-kk-src+target + - translate_as_en-kk-source+target reference: '' - 38e6b64b-eb3c-4efc-aca4-7172784804fc: !Template + 26a42802-9354-4fce-9f6b-e44819534b76: !Template answer_choices: null - id: 38e6b64b-eb3c-4efc-aca4-7172784804fc - jinja: 'Given the following passage: {{translation["en"]}} , a good Kazakh translation - is: ||| {{translation["kk"]}}' + id: 26a42802-9354-4fce-9f6b-e44819534b76 + jinja: 'What is the English translation of the Kazakh sentence: {{translation["kk"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-kk-target - reference: '' - 43cf4166-7da6-42fd-9837-067646fbdc5a: !Template + - gpt-3-kk-en-source+target + reference: GPT-3 paper + 56b3c419-8774-4c38-aae0-969f3ce948cb: !Template answer_choices: null - id: 43cf4166-7da6-42fd-9837-067646fbdc5a - jinja: 'Translate Kazakh into English: {{translation["kk"]}} + id: 56b3c419-8774-4c38-aae0-969f3ce948cb + jinja: 'If the Kazakh version says: {{translation["kk"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-kk-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 6c6a16ee-80d9-4b2a-a087-47b741344eb3: !Template + - version-kk-en-source+target + reference: '' + 5c70e3bc-ae9d-448c-951b-067bc459e666: !Template answer_choices: null - id: 6c6a16ee-80d9-4b2a-a087-47b741344eb3 - jinja: 'Given the following passage: {{translation["kk"]}} , a good English translation - is: + id: 5c70e3bc-ae9d-448c-951b-067bc459e666 + jinja: 'Translate into Kazakh: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-kk-en-target + - translate-en-kk-target reference: '' - 70054e8a-8c17-4fe3-a89e-af22f97d0690: !Template + 65e01c0f-1b92-490a-a0e3-95f8d65c0897: !Template answer_choices: null - id: 70054e8a-8c17-4fe3-a89e-af22f97d0690 - jinja: '{{translation["kk"]}} translates into English as: + id: 65e01c0f-1b92-490a-a0e3-95f8d65c0897 + jinja: 'Translate English into Kazakh: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_kk-en-target + - translate-en-kk-source+starget reference: '' - 710acdc8-6990-4d37-a6d1-53d17d6744d6: !Template + 6a379888-9584-4fde-beb6-ac021d7f3927: !Template answer_choices: null - id: 710acdc8-6990-4d37-a6d1-53d17d6744d6 - jinja: 'What is the Kazakh translation of: {{translation["en"]}} + id: 6a379888-9584-4fde-beb6-ac021d7f3927 + jinja: 'If the original version says: {{translation["en"]}}; then the Kazakh version + should say: - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-kk-target + - version-en-kk-target reference: '' - 74a834e6-df41-40b8-99f6-015b4285d6c6: !Template + 6a88262a-bb53-489c-b6d9-7439a5db4dd9: !Template answer_choices: null - id: 74a834e6-df41-40b8-99f6-015b4285d6c6 - jinja: 'Translate English into Kazakh: {{translation["en"]}} + id: 6a88262a-bb53-489c-b6d9-7439a5db4dd9 + jinja: 'What is the English translation of : {{translation["kk"]}} - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-kk-src+starget + - gpt-3-kk-en-target + reference: GPT-3 paper + 6c0c700f-76c2-4673-ad31-a24bde943eb1: !Template + answer_choices: null + id: 6c0c700f-76c2-4673-ad31-a24bde943eb1 + jinja: 'How do you say {{translation["en"]}} in Kazakh? + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-kk-target reference: '' - 8f2e8da8-3864-4a93-8362-a82f4718f842: !Template + 724de4bc-efb7-4203-a82d-983a38d2802e: !Template answer_choices: null - id: 8f2e8da8-3864-4a93-8362-a82f4718f842 - jinja: 'Kazakh: {{translation["kk"]}} translates into English as: + id: 724de4bc-efb7-4203-a82d-983a38d2802e + jinja: 'Translate into English: {{translation["kk"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_kk-en-source+target + - translate-kk-en-target reference: '' - 9158c439-cc8d-40a2-a692-693564c91db3: !Template + 7dd8375e-4100-4030-92c0-8fdd4b2d04ba: !Template answer_choices: null - id: 9158c439-cc8d-40a2-a692-693564c91db3 - jinja: 'If the English version says: {{translation["en"]}}; then the Kazakh version - should say: + id: 7dd8375e-4100-4030-92c0-8fdd4b2d04ba + jinja: 'Translate Kazakh into English: {{translation["kk"]}} - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-kk-source+target + - translate-kk-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 88171545-f09a-40d1-b9cc-4e391a23f0f8: !Template + answer_choices: null + id: 88171545-f09a-40d1-b9cc-4e391a23f0f8 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Kazakh translation is: + + ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-kk-source+target reference: '' - 9389ace0-7c38-445a-be50-0578249166b5: !Template + a1638c3a-8190-4a18-8384-4b0d20cf975e: !Template answer_choices: null - id: 9389ace0-7c38-445a-be50-0578249166b5 - jinja: 'If the Kazakh version says: {{translation["kk"]}}; then the English version - should say: + id: a1638c3a-8190-4a18-8384-4b0d20cf975e + jinja: 'If the original version says: {{translation["kk"]}}; then the English + version should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -185,11 +214,24 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-kk-en-source+target + - version-kk-en-target reference: '' - a057638b-49b5-414e-a63b-3e70f795ea5f: !Template + b52dee88-79ee-4a7c-876e-3f71764a8aee: !Template answer_choices: null - id: a057638b-49b5-414e-a63b-3e70f795ea5f + id: b52dee88-79ee-4a7c-876e-3f71764a8aee + jinja: 'Given the following source text in Kazakh: {{translation["kk"]}} , a good + English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-kk-en-source+target + reference: '' + b56ca813-ce38-44c9-86e2-160e22f4dc1e: !Template + answer_choices: null + id: b56ca813-ce38-44c9-86e2-160e22f4dc1e jinja: 'How do you say {{translation["kk"]}} in English? ||| {{translation["en"]}}' @@ -201,39 +243,37 @@ templates: name: !!python/tuple - how_to_say-kk-en-target reference: '' - afaa595f-deae-4e35-917c-d58151ec6d18: !Template + c56d3846-cc9b-41a0-aa4f-a1beedd65ec2: !Template answer_choices: null - id: afaa595f-deae-4e35-917c-d58151ec6d18 - jinja: 'Translate into English: {{translation["kk"]}} + id: c56d3846-cc9b-41a0-aa4f-a1beedd65ec2 + jinja: 'English: {{translation["en"]}} = Kazakh: - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-kk-en-target - reference: '' - b101e4db-6b02-436c-9be0-d2fee827576f: !Template + - xglm-en-kk-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + db0dd18d-e620-442d-9dbd-10bfebd1d433: !Template answer_choices: null - id: b101e4db-6b02-436c-9be0-d2fee827576f - jinja: 'What is the English translation of : {{translation["kk"]}} - - ||| {{translation["en"]}}' + id: db0dd18d-e620-442d-9dbd-10bfebd1d433 + jinja: 'Given the following passage: {{translation["en"]}} , a good Kazakh translation + is: ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-kk-en-target + - a_good_translation-en-kk-target reference: '' - d11653e9-8fec-430e-9a97-f2195fb71c02: !Template + e2896e24-9b0b-466d-989f-f0ed6ab8842a: !Template answer_choices: null - id: d11653e9-8fec-430e-9a97-f2195fb71c02 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Kazakh translation is: + id: e2896e24-9b0b-466d-989f-f0ed6ab8842a + jinja: ' {{translation["en"]}} translates into Kazakh as: ||| {{translation["kk"]}}' metadata: !TemplateMetadata @@ -242,12 +282,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-kk-source+target + - translate_as_en-kk-target reference: '' - d3ce23e8-fad3-4473-8087-da01fb9af040: !Template + e31ce0bc-2ac2-46de-8254-d9c3ed665c18: !Template answer_choices: null - id: d3ce23e8-fad3-4473-8087-da01fb9af040 - jinja: 'Translate into Kazakh: {{translation["en"]}} + id: e31ce0bc-2ac2-46de-8254-d9c3ed665c18 + jinja: 'What is the Kazakh translation of: {{translation["en"]}} ||| {{translation["kk"]}}' metadata: !TemplateMetadata @@ -256,53 +296,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-kk-target - reference: '' - d6ea1138-feed-499b-91a1-328483fabc1f: !Template + - gpt-3-en-kk-target + reference: GPT-3 paper + e3d7e25f-3fc1-44ec-8974-1c89d7976e15: !Template answer_choices: null - id: d6ea1138-feed-499b-91a1-328483fabc1f - jinja: ' {{translation["en"]}} translates into Kazakh as: + id: e3d7e25f-3fc1-44ec-8974-1c89d7976e15 + jinja: '{{translation["kk"]}} = English: - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-kk-target - reference: '' - d7e61faa-02f8-4801-9f61-c2425b8dddf8: !Template + - xglm-kk-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e568db81-1c82-4e2f-bd74-d86a67ce4fc9: !Template answer_choices: null - id: d7e61faa-02f8-4801-9f61-c2425b8dddf8 - jinja: 'Given the following source text in Kazakh: {{translation["kk"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: e568db81-1c82-4e2f-bd74-d86a67ce4fc9 + jinja: 'What is the Kazakh translation of the English sentence: {{translation["en"]}} + + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-kk-en-source+target - reference: '' - d80b19a0-d876-4d51-a479-c5a25a640113: !Template + - gpt-3-en-kk-source+target + reference: GPT 3 paper + f0cb7953-a0a6-4f13-8023-bbb2dffe259e: !Template answer_choices: null - id: d80b19a0-d876-4d51-a479-c5a25a640113 - jinja: 'What is the English translation of the Kazakh sentence: {{translation["kk"]}} + id: f0cb7953-a0a6-4f13-8023-bbb2dffe259e + jinja: 'Kazakh: {{translation["kk"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-kk-en-src+target + - translate_as_kk-en-source+target reference: '' - e94d40e1-5d03-444e-a814-5158a819842d: !Template + f2c0559c-5768-432a-b5dc-b9aba5577038: !Template answer_choices: null - id: e94d40e1-5d03-444e-a814-5158a819842d - jinja: 'English: {{translation["en"]}} translates into Kazakh as: + id: f2c0559c-5768-432a-b5dc-b9aba5577038 + jinja: '{{translation["en"]}} = Kazakh: ||| {{translation["kk"]}}' metadata: !TemplateMetadata @@ -311,5 +352,20 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-kk-source+target + - xglm-en-kk-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f7171f67-3c41-44f8-8f75-c5958ea7811f: !Template + answer_choices: null + id: f7171f67-3c41-44f8-8f75-c5958ea7811f + jinja: 'Given the following passage: {{translation["kk"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-kk-en-target reference: '' diff --git a/promptsource/templates/wmt19/lt-en/templates.yaml b/promptsource/templates/wmt19/lt-en/templates.yaml index 400d48977..b28e3fa50 100644 --- a/promptsource/templates/wmt19/lt-en/templates.yaml +++ b/promptsource/templates/wmt19/lt-en/templates.yaml @@ -1,52 +1,79 @@ dataset: wmt19 subset: lt-en templates: - 118f65c0-dd02-4abb-a66f-47d3d8799a07: !Template + 09d912d9-54b5-4094-886a-78e9d3622c49: !Template answer_choices: null - id: 118f65c0-dd02-4abb-a66f-47d3d8799a07 - jinja: 'How do you say {{translation["lt"]}} in English? + id: 09d912d9-54b5-4094-886a-78e9d3622c49 + jinja: 'Given the following passage: {{translation["en"]}} , a good Lithuanian + translation is: ||| {{translation["lt"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-lt-target + reference: '' + 1081536f-f4d4-47b5-9b08-e75756182e0e: !Template + answer_choices: null + id: 1081536f-f4d4-47b5-9b08-e75756182e0e + jinja: 'If the Lithuanian version says: {{translation["lt"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-lt-en-target + - version-lt-en-source+target reference: '' - 119b07b6-d337-486c-9b6b-65a571e5821e: !Template + 17bcfc14-24c3-410e-861a-9758751339f1: !Template answer_choices: null - id: 119b07b6-d337-486c-9b6b-65a571e5821e - jinja: 'Translate into Lithuanian: {{translation["en"]}} + id: 17bcfc14-24c3-410e-861a-9758751339f1 + jinja: 'Lithuanian: {{translation["lt"]}} translates into English as: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lt-target + - translate_as_lt-en-source+target reference: '' - 11a563fa-7dc4-4902-bc8f-437f49103165: !Template + 19f026f2-7bf7-48e7-93f8-d711e746bbbd: !Template answer_choices: null - id: 11a563fa-7dc4-4902-bc8f-437f49103165 - jinja: 'If the English version says: {{translation["en"]}}; then the Lithuanian - version should say: + id: 19f026f2-7bf7-48e7-93f8-d711e746bbbd + jinja: 'English: {{translation["en"]}} translates into Lithuanian as: - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lt-source+target + - translate_as_en-lt-source+target reference: '' - 1d5cea12-e7f1-4eed-9568-dc0d029ae09c: !Template + 1d31906c-be02-47a3-926f-0bd9c4bf05c9: !Template + answer_choices: null + id: 1d31906c-be02-47a3-926f-0bd9c4bf05c9 + jinja: 'What is the English translation of the Lithuanian sentence: {{translation["lt"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-lt-en-source+target + reference: GPT-3 paper + 239dec95-271d-4301-8cab-fca10958c97f: !Template answer_choices: null - id: 1d5cea12-e7f1-4eed-9568-dc0d029ae09c + id: 239dec95-271d-4301-8cab-fca10958c97f jinja: ' {{translation["en"]}} translates into Lithuanian as: ||| {{translation["lt"]}}' @@ -58,24 +85,24 @@ templates: name: !!python/tuple - translate_as_en-lt-target reference: '' - 1d7db603-5023-4363-808d-da1720fe0e6d: !Template + 2486e08b-e932-4f7c-b2fe-a5ff88e4a5a4: !Template answer_choices: null - id: 1d7db603-5023-4363-808d-da1720fe0e6d - jinja: 'Given the following source text in Lithuanian: {{translation["lt"]}} , - a good English translation is: ||| {{translation["en"]}}' + id: 2486e08b-e932-4f7c-b2fe-a5ff88e4a5a4 + jinja: 'Translate into Lithuanian: {{translation["en"]}} + + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lt-en-source+target + - translate-en-lt-target reference: '' - 4838300a-6622-44fb-a72d-44f75fe1f4fe: !Template + 388c481e-9ed2-4860-87b0-00a32b3f5a4e: !Template answer_choices: null - id: 4838300a-6622-44fb-a72d-44f75fe1f4fe - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Lithuanian translation is: + id: 388c481e-9ed2-4860-87b0-00a32b3f5a4e + jinja: '{{translation["en"]}} = Lithuanian: ||| {{translation["lt"]}}' metadata: !TemplateMetadata @@ -84,26 +111,42 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lt-source+target + - xglm-en-lt-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3cea9425-dbaa-4c4c-a1b0-133683bcca53: !Template + answer_choices: null + id: 3cea9425-dbaa-4c4c-a1b0-133683bcca53 + jinja: '{{translation["lt"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_lt-en-target reference: '' - 4c90fb04-938c-44a4-98da-f8e049bc58a3: !Template + 48ed0004-1cf3-42e2-854c-756a73d99eb9: !Template answer_choices: null - id: 4c90fb04-938c-44a4-98da-f8e049bc58a3 - jinja: 'What is the Lithuanian translation of the English sentence: {{translation["en"]}} + id: 48ed0004-1cf3-42e2-854c-756a73d99eb9 + jinja: 'If the original version says: {{translation["en"]}}; then the Lithuanian + version should say: - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lt-src+target + - version-en-lt-target reference: '' - 62b686d3-b82b-4399-8f8b-3f51977b0b6c: !Template + 604d7587-c23e-443e-ab71-4c06e55f1463: !Template answer_choices: null - id: 62b686d3-b82b-4399-8f8b-3f51977b0b6c - jinja: 'English: {{translation["en"]}} translates into Lithuanian as: + id: 604d7587-c23e-443e-ab71-4c06e55f1463 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Lithuanian translation is: ||| {{translation["lt"]}}' metadata: !TemplateMetadata @@ -112,11 +155,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lt-source+target + - a_good_translation-en-lt-source+target reference: '' - 65576b55-81ec-4761-8774-74a30eace165: !Template + 7006b213-cc36-4153-ae89-b85a73ec598e: !Template answer_choices: null - id: 65576b55-81ec-4761-8774-74a30eace165 + id: 7006b213-cc36-4153-ae89-b85a73ec598e jinja: 'What is the English translation of : {{translation["lt"]}} ||| {{translation["en"]}}' @@ -127,11 +170,24 @@ templates: original_task: true name: !!python/tuple - gpt-3-lt-en-target + reference: GPT-3 paper + 79d4fccc-593b-4dce-b6ef-3b3c6c189de4: !Template + answer_choices: null + id: 79d4fccc-593b-4dce-b6ef-3b3c6c189de4 + jinja: 'Given the following source text in Lithuanian: {{translation["lt"]}} , + a good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-lt-en-source+target reference: '' - 66774f5f-8435-41b6-9597-706c190def94: !Template + 8ba668c5-9ad8-4718-aa54-2e5d8b746a73: !Template answer_choices: null - id: 66774f5f-8435-41b6-9597-706c190def94 - jinja: 'Lithuanian: {{translation["lt"]}} translates into English as: + id: 8ba668c5-9ad8-4718-aa54-2e5d8b746a73 + jinja: '{{translation["lt"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -140,41 +196,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_lt-en-source+target - reference: '' - 693fc37a-a7e8-47d7-887c-008e193d9746: !Template + - xglm-lt-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9d160f00-e1d9-477c-b71a-4ded861367f9: !Template answer_choices: null - id: 693fc37a-a7e8-47d7-887c-008e193d9746 - jinja: 'Translate into English: {{translation["lt"]}} + id: 9d160f00-e1d9-477c-b71a-4ded861367f9 + jinja: 'Lithuanian: {{translation["lt"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lt-en-target - reference: '' - 7ae534d1-5beb-4b4c-923f-1b748f4cc1bc: !Template + - xglm-lt-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c342a4ea-e09b-445c-96f5-c79341d5cb40: !Template answer_choices: null - id: 7ae534d1-5beb-4b4c-923f-1b748f4cc1bc - jinja: 'If the Lithuanian version says: {{translation["lt"]}}; then the English - version should say: + id: c342a4ea-e09b-445c-96f5-c79341d5cb40 + jinja: 'How do you say {{translation["en"]}} in Lithuanian? - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lt-en-source+target + - how_to_say-en-lt-target reference: '' - a1c2cbeb-86eb-419c-9bd9-6579a513391f: !Template + c978e375-5b34-45a8-87aa-22b51cccb868: !Template answer_choices: null - id: a1c2cbeb-86eb-419c-9bd9-6579a513391f - jinja: 'What is the Lithuanian translation of: {{translation["en"]}} + id: c978e375-5b34-45a8-87aa-22b51cccb868 + jinja: 'What is the Lithuanian translation of the English sentence: {{translation["en"]}} ||| {{translation["lt"]}}' metadata: !TemplateMetadata @@ -183,110 +238,112 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lt-target - reference: '' - a50239e7-e544-4a1a-a3e1-dd58647879fa: !Template + - gpt-3-en-lt-source+target + reference: GPT 3 paper + cbc63670-78c7-45d1-9ceb-451b9c22f31d: !Template answer_choices: null - id: a50239e7-e544-4a1a-a3e1-dd58647879fa - jinja: 'Given the following passage: {{translation["lt"]}} , a good English translation - is: + id: cbc63670-78c7-45d1-9ceb-451b9c22f31d + jinja: 'Translate into English: {{translation["lt"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lt-en-target + - translate-lt-en-target reference: '' - a5bc4337-60ea-464e-8ddd-f8595f34cc4f: !Template + cf64d506-a054-49cf-83f9-57eb3923680e: !Template answer_choices: null - id: a5bc4337-60ea-464e-8ddd-f8595f34cc4f - jinja: '{{translation["lt"]}} translates into English as: + id: cf64d506-a054-49cf-83f9-57eb3923680e + jinja: 'If the English version says: {{translation["en"]}}; then the Lithuanian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_lt-en-target + - version-en-lt-source+target reference: '' - b278a391-1bc3-42df-aabd-4891106bdf18: !Template + d1951cc8-cef9-49ab-9df1-a9a58caa38fc: !Template answer_choices: null - id: b278a391-1bc3-42df-aabd-4891106bdf18 - jinja: 'How do you say {{translation["en"]}} in Lithuanian? + id: d1951cc8-cef9-49ab-9df1-a9a58caa38fc + jinja: 'What is the Lithuanian translation of: {{translation["en"]}} - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-lt-target - reference: '' - c48fb5f0-4b98-4d22-b3de-b5b9d6d06a46: !Template + - gpt-3-en-lt-target + reference: GPT-3 paper + dd1e7adb-8331-4a0d-844e-91076648f25a: !Template answer_choices: null - id: c48fb5f0-4b98-4d22-b3de-b5b9d6d06a46 - jinja: 'Translate English into Lithuanian: {{translation["en"]}} + id: dd1e7adb-8331-4a0d-844e-91076648f25a + jinja: 'Given the following passage: {{translation["lt"]}} , a good English translation + is: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lt-src+starget + - a_good_translation-lt-en-target reference: '' - c8f476e6-75ff-468d-9fc9-69aea45c7054: !Template + de291c64-3989-4104-8c50-655c2fa73969: !Template answer_choices: null - id: c8f476e6-75ff-468d-9fc9-69aea45c7054 - jinja: 'If the original version says: {{translation["en"]}}; then the Lithuanian + id: de291c64-3989-4104-8c50-655c2fa73969 + jinja: 'If the original version says: {{translation["lt"]}}; then the English version should say: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lt-target + - version-lt-en-target reference: '' - d9ef689f-bd41-483d-b2a6-6fa68a90dd07: !Template + dec5fe1c-5d8a-477d-8a63-cdbf1d0374f1: !Template answer_choices: null - id: d9ef689f-bd41-483d-b2a6-6fa68a90dd07 - jinja: 'What is the English translation of the Lithuanian sentence: {{translation["lt"]}} + id: dec5fe1c-5d8a-477d-8a63-cdbf1d0374f1 + jinja: 'Translate English into Lithuanian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lt-en-src+target + - translate-en-lt-source+starget reference: '' - f0fee957-425b-472e-9ef0-862ccdc1dd8e: !Template + ea94c785-cff2-47f5-a06d-f31ce7885b7f: !Template answer_choices: null - id: f0fee957-425b-472e-9ef0-862ccdc1dd8e - jinja: 'Given the following passage: {{translation["en"]}} , a good Lithuanian - translation is: ||| {{translation["lt"]}}' + id: ea94c785-cff2-47f5-a06d-f31ce7885b7f + jinja: 'How do you say {{translation["lt"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lt-target + - how_to_say-lt-en-target reference: '' - f63651e7-2ab4-424c-87ed-aa8e809b950e: !Template + f777af07-abeb-48bb-ad9d-16f490dd1039: !Template answer_choices: null - id: f63651e7-2ab4-424c-87ed-aa8e809b950e + id: f777af07-abeb-48bb-ad9d-16f490dd1039 jinja: 'Translate Lithuanian into English: {{translation["lt"]}} ||| {{translation["en"]}}' @@ -298,18 +355,17 @@ templates: name: !!python/tuple - translate-lt-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - ff80e819-392d-4656-a75c-866d74ce5d33: !Template + fb7d19ad-16ac-42bf-93a8-3960652f98c9: !Template answer_choices: null - id: ff80e819-392d-4656-a75c-866d74ce5d33 - jinja: 'If the original version says: {{translation["lt"]}}; then the English - version should say: + id: fb7d19ad-16ac-42bf-93a8-3960652f98c9 + jinja: 'English: {{translation["en"]}} = Lithuanian: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lt-en-target - reference: '' + - xglm-en-lt-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt19/ru-en/templates.yaml b/promptsource/templates/wmt19/ru-en/templates.yaml index 81ce55fdf..a84c7ddad 100644 --- a/promptsource/templates/wmt19/ru-en/templates.yaml +++ b/promptsource/templates/wmt19/ru-en/templates.yaml @@ -1,25 +1,25 @@ dataset: wmt19 subset: ru-en templates: - 2222898b-a719-45b2-bd77-78f9ff7f7d8f: !Template + 077da5bd-4b32-44ed-ac04-d0530e1a2364: !Template answer_choices: null - id: 2222898b-a719-45b2-bd77-78f9ff7f7d8f - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 077da5bd-4b32-44ed-ac04-d0530e1a2364 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target + - a_good_translation-en-ru-source+target reference: '' - 25e8fb3b-9612-443e-a6a9-b4793d8f0cca: !Template + 09c29b29-61f1-43a1-9ea2-67c2c19fcbe3: !Template answer_choices: null - id: 25e8fb3b-9612-443e-a6a9-b4793d8f0cca - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 09c29b29-61f1-43a1-9ea2-67c2c19fcbe3 + jinja: ' {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -28,70 +28,67 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target + - translate_as_en-ru-target reference: '' - 347eb46a-4b18-4aa9-bac8-748ec9481be4: !Template + 0d95df60-a164-4811-b368-53b1d6c3b164: !Template answer_choices: null - id: 347eb46a-4b18-4aa9-bac8-748ec9481be4 - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 0d95df60-a164-4811-b368-53b1d6c3b164 + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: '' - 3ff8be77-d920-4f79-8f81-64b022744ce4: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 17c15ac0-5839-491a-bcf3-156077363824: !Template answer_choices: null - id: 3ff8be77-d920-4f79-8f81-64b022744ce4 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 17c15ac0-5839-491a-bcf3-156077363824 + jinja: 'English: {{translation["en"]}} translates into Russian as: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - translate_as_en-ru-source+target reference: '' - 493e3602-1bf2-473f-8dca-37f0064dfa86: !Template + 19d521d8-dfdb-494e-b91a-98193d6e2739: !Template answer_choices: null - id: 493e3602-1bf2-473f-8dca-37f0064dfa86 - jinja: ' {{translation["en"]}} translates into Russian as: - - ||| {{translation["ru"]}}' + id: 19d521d8-dfdb-494e-b91a-98193d6e2739 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - a_good_translation-ru-en-source+target reference: '' - 5433c9ee-ed73-4660-b9e6-d9be4f7c70e4: !Template + 3043dadc-4d0d-4b75-962d-5de834da539d: !Template answer_choices: null - id: 5433c9ee-ed73-4660-b9e6-d9be4f7c70e4 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 3043dadc-4d0d-4b75-962d-5de834da539d + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target - reference: '' - 64ceab1f-773e-4c4d-a721-c5964d86fffe: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 315a0c39-2d00-4b04-822d-477c5cf09164: !Template answer_choices: null - id: 64ceab1f-773e-4c4d-a721-c5964d86fffe - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 315a0c39-2d00-4b04-822d-477c5cf09164 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -100,55 +97,68 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: '' - 6a134c4c-d73a-480c-bfdc-4b180c253d87: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 3b841603-c2b6-4b1f-8f79-93426cf56bc3: !Template answer_choices: null - id: 6a134c4c-d73a-480c-bfdc-4b180c253d87 - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + id: 3b841603-c2b6-4b1f-8f79-93426cf56bc3 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - version-en-ru-source+target reference: '' - 6cef404a-7c3c-47d0-8782-ea3b74cecc68: !Template + 3b8e645f-0658-4cc5-8e31-1dd43a90d2da: !Template answer_choices: null - id: 6cef404a-7c3c-47d0-8782-ea3b74cecc68 - jinja: '{{translation["ru"]}} translates into English as: + id: 3b8e645f-0658-4cc5-8e31-1dd43a90d2da + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - gpt-3-en-ru-target + reference: GPT-3 paper + 4c399c8b-52e3-4720-9aca-aba5f7345148: !Template + answer_choices: null + id: 4c399c8b-52e3-4720-9aca-aba5f7345148 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-ru-target reference: '' - 6d92111c-1aff-46d3-bec5-38a5edeebe77: !Template + 551f97bf-1d7a-4400-bcb6-f79a4c9c3a8f: !Template answer_choices: null - id: 6d92111c-1aff-46d3-bec5-38a5edeebe77 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 551f97bf-1d7a-4400-bcb6-f79a4c9c3a8f + jinja: 'How do you say {{translation["en"]}} in Russian? - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-src+starget + - how_to_say-en-ru-target reference: '' - 738d2f5f-1e6a-4cae-814e-cef6368d37ca: !Template + 66fbd36a-710b-4277-8859-2cdf7fc75982: !Template answer_choices: null - id: 738d2f5f-1e6a-4cae-814e-cef6368d37ca - jinja: 'How do you say {{translation["ru"]}} in English? + id: 66fbd36a-710b-4277-8859-2cdf7fc75982 + jinja: 'Russian: {{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -157,24 +167,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate_as_ru-en-source+target reference: '' - 76d1e5dc-ed29-4c66-83a9-1627e68ac0b8: !Template + 78c33a2d-6895-491a-abce-ec474193048d: !Template answer_choices: null - id: 76d1e5dc-ed29-4c66-83a9-1627e68ac0b8 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 78c33a2d-6895-491a-abce-ec474193048d + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - version-en-ru-target + reference: '' + 81e99ccc-91f9-45e2-94c4-add17ed68ef4: !Template + answer_choices: null + id: 81e99ccc-91f9-45e2-94c4-add17ed68ef4 + jinja: 'Translate into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-ru-en-target reference: '' - 8a8b26ec-5996-460a-826d-c87b1a38def3: !Template + 8934e560-7bed-4718-8864-4356aae19c72: !Template answer_choices: null - id: 8a8b26ec-5996-460a-826d-c87b1a38def3 + id: 8934e560-7bed-4718-8864-4356aae19c72 jinja: 'Translate into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' @@ -186,10 +212,38 @@ templates: name: !!python/tuple - translate-en-ru-target reference: '' - 8ab2840e-4df0-4cd5-9ac4-21c8d84af78b: !Template + 90ed694d-fa0a-45d3-b300-89675e37fa85: !Template answer_choices: null - id: 8ab2840e-4df0-4cd5-9ac4-21c8d84af78b - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 90ed694d-fa0a-45d3-b300-89675e37fa85 + jinja: 'English: {{translation["en"]}} = Russian: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 9913cc00-d35e-42ec-807e-b32df461d0be: !Template + answer_choices: null + id: 9913cc00-d35e-42ec-807e-b32df461d0be + jinja: 'Russian: {{translation["ru"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a719f99f-bcc5-472c-923b-47138d0be28e: !Template + answer_choices: null + id: a719f99f-bcc5-472c-923b-47138d0be28e + jinja: 'What is the English translation of : {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -198,28 +252,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 8c1c4f53-e5a1-4131-8adc-053944fb76a9: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + ae34ab89-2e9c-4f95-ac8c-0cdd5361329c: !Template answer_choices: null - id: 8c1c4f53-e5a1-4131-8adc-053944fb76a9 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: ae34ab89-2e9c-4f95-ac8c-0cdd5361329c + jinja: 'Translate English into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - translate-en-ru-source+starget reference: '' - 98d4bc66-c822-44cc-a8bc-325c429f48d3: !Template + b545a1ed-be0b-433e-9c51-ef902d345a4d: !Template answer_choices: null - id: 98d4bc66-c822-44cc-a8bc-325c429f48d3 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: b545a1ed-be0b-433e-9c51-ef902d345a4d + jinja: '{{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -228,81 +280,85 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target - reference: '' - a29bb5d6-fa33-4334-890c-dd565becbfd9: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c5f14676-23d8-4ef1-aa37-271db7a27153: !Template answer_choices: null - id: a29bb5d6-fa33-4334-890c-dd565becbfd9 - jinja: 'Translate into English: {{translation["ru"]}} + id: c5f14676-23d8-4ef1-aa37-271db7a27153 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - a_good_translation-ru-en-target reference: '' - b3025df4-de0b-4cc9-90ca-3acfc31e3af8: !Template + dba4eace-b907-437a-abf7-c0d173a24711: !Template answer_choices: null - id: b3025df4-de0b-4cc9-90ca-3acfc31e3af8 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: dba4eace-b907-437a-abf7-c0d173a24711 + jinja: 'Translate Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target - reference: '' - d081ca93-c999-41f4-b9d6-d9e5ba83b02e: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e036a9fa-5836-4c97-b52d-677d07092097: !Template answer_choices: null - id: d081ca93-c999-41f4-b9d6-d9e5ba83b02e - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: e036a9fa-5836-4c97-b52d-677d07092097 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - version-ru-en-source+target reference: '' - d8f8de49-13ba-42a9-ab3f-5456c64ff59c: !Template + ed2b0b2f-f79c-4f15-b8ff-0b5d42e4292b: !Template answer_choices: null - id: d8f8de49-13ba-42a9-ab3f-5456c64ff59c - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: ed2b0b2f-f79c-4f15-b8ff-0b5d42e4292b + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-src+target + - translate_as_ru-en-target reference: '' - eb5d5176-d9e6-4172-a951-84ba5d86eee6: !Template + ee346d61-67a2-4cc4-adff-6cd71eab5a16: !Template answer_choices: null - id: eb5d5176-d9e6-4172-a951-84ba5d86eee6 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: ee346d61-67a2-4cc4-adff-6cd71eab5a16 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-src+target + - version-ru-en-target reference: '' - ef943a36-5dd4-407e-96bb-04a3298c92d9: !Template + fae13413-02db-4c10-9b5b-43ccd35172a7: !Template answer_choices: null - id: ef943a36-5dd4-407e-96bb-04a3298c92d9 - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: fae13413-02db-4c10-9b5b-43ccd35172a7 + jinja: 'How do you say {{translation["ru"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - how_to_say-ru-en-target reference: '' diff --git a/promptsource/templates/wmt19/zh-en/templates.yaml b/promptsource/templates/wmt19/zh-en/templates.yaml index d5fedaa22..682684dc1 100644 --- a/promptsource/templates/wmt19/zh-en/templates.yaml +++ b/promptsource/templates/wmt19/zh-en/templates.yaml @@ -1,81 +1,80 @@ dataset: wmt19 subset: zh-en templates: - 10d9af5d-65ad-4744-93f5-5b4ce3ff8355: !Template + 036ae20f-5a46-410e-8aae-2345539dea64: !Template answer_choices: null - id: 10d9af5d-65ad-4744-93f5-5b4ce3ff8355 - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: 036ae20f-5a46-410e-8aae-2345539dea64 + jinja: 'Translate English into Chinese: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target + - translate-en-zh-source+starget reference: '' - 469e173c-949f-470b-819b-eec2851b2138: !Template + 1a2cbf2b-0369-4176-b9ea-aa49995db87e: !Template answer_choices: null - id: 469e173c-949f-470b-819b-eec2851b2138 - jinja: 'Translate English into Chinese: {{translation["en"]}} + id: 1a2cbf2b-0369-4176-b9ea-aa49995db87e + jinja: 'Translate Chinese into English: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-src+starget - reference: '' - 500e5a0b-5bd3-42b6-b5fb-a9b16cce4803: !Template + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 24eb3c84-a760-4493-b1b3-5501eba6a1c9: !Template answer_choices: null - id: 500e5a0b-5bd3-42b6-b5fb-a9b16cce4803 - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: 24eb3c84-a760-4493-b1b3-5501eba6a1c9 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target + - version-zh-en-target reference: '' - 551f3d20-d549-4c3c-b987-18e47fc05802: !Template + 25871786-1c7a-4cb3-a1a3-e6b0e7137e02: !Template answer_choices: null - id: 551f3d20-d549-4c3c-b987-18e47fc05802 - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese - version should say: + id: 25871786-1c7a-4cb3-a1a3-e6b0e7137e02 + jinja: 'English: {{translation["en"]}} translates into Chinese as: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target + - translate_as_en-zh-source+target reference: '' - 62886e69-7b80-4730-b047-4bace183bad7: !Template + 28959a82-6a8c-4e2f-be8e-143e3f9c36fc: !Template answer_choices: null - id: 62886e69-7b80-4730-b047-4bace183bad7 - jinja: 'Translate into Chinese: {{translation["en"]}} + id: 28959a82-6a8c-4e2f-be8e-143e3f9c36fc + jinja: 'How do you say {{translation["en"]}} in Chinese? - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - how_to_say-en-zh-target reference: '' - 6485f138-7c13-4212-ad22-8015c84a2cf4: !Template + 2f41e2db-f1d3-4ee0-bce2-ecd3a62b2ad7: !Template answer_choices: null - id: 6485f138-7c13-4212-ad22-8015c84a2cf4 + id: 2f41e2db-f1d3-4ee0-bce2-ecd3a62b2ad7 jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version should say: @@ -88,51 +87,52 @@ templates: name: !!python/tuple - version-en-zh-source+target reference: '' - 6754d78e-d4d1-4812-8fb6-d4e570933622: !Template + 2f80c39c-5bba-4295-9175-9f0f72622045: !Template answer_choices: null - id: 6754d78e-d4d1-4812-8fb6-d4e570933622 - jinja: ' {{translation["en"]}} translates into Chinese as: + id: 2f80c39c-5bba-4295-9175-9f0f72622045 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-target - reference: '' - 6dec7703-89bc-4b43-b8a5-d56fa068a97c: !Template + - gpt-3-en-zh-source+target + reference: GPT 3 paper + 30aed8d9-b822-42f9-a1b1-b18537b9df60: !Template answer_choices: null - id: 6dec7703-89bc-4b43-b8a5-d56fa068a97c - jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 30aed8d9-b822-42f9-a1b1-b18537b9df60 + jinja: '{{translation["zh"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-source+target - reference: '' - 6ea3018a-4c5f-47c0-9d8c-7879acc7484a: !Template + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 32758794-35ba-44c4-b7cc-3dd1076b33e9: !Template answer_choices: null - id: 6ea3018a-4c5f-47c0-9d8c-7879acc7484a - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: 32758794-35ba-44c4-b7cc-3dd1076b33e9 + jinja: 'Translate into Chinese: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target + - translate-en-zh-target reference: '' - 716e1c1a-0bee-4293-ac2e-0e993f810fcf: !Template + 327b953c-13c0-4ee5-b33b-15762767d2b8: !Template answer_choices: null - id: 716e1c1a-0bee-4293-ac2e-0e993f810fcf - jinja: '{{translation["zh"]}} translates into English as: + id: 327b953c-13c0-4ee5-b33b-15762767d2b8 + jinja: 'How do you say {{translation["zh"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -141,39 +141,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target + - how_to_say-zh-en-target reference: '' - 739ab695-96ce-450e-b1a4-cac16732210e: !Template + 44148f50-2f94-4505-8fd2-8b8524f61f69: !Template answer_choices: null - id: 739ab695-96ce-450e-b1a4-cac16732210e - jinja: 'How do you say {{translation["zh"]}} in English? - - ||| {{translation["en"]}}' + id: 44148f50-2f94-4505-8fd2-8b8524f61f69 + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target + - a_good_translation-en-zh-target reference: '' - 75084cb1-54f6-4fd1-ad11-be0176766173: !Template + 4a4e6ef6-79fb-4b59-a471-6caad2450434: !Template answer_choices: null - id: 75084cb1-54f6-4fd1-ad11-be0176766173 - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: 4a4e6ef6-79fb-4b59-a471-6caad2450434 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-src+target + - a_good_translation-en-zh-source+target reference: '' - 7becef8b-011a-4c22-9159-dd755c749d27: !Template + 6c773fcf-3fd9-4a07-b0cc-2a209421666d: !Template answer_choices: null - id: 7becef8b-011a-4c22-9159-dd755c749d27 + id: 6c773fcf-3fd9-4a07-b0cc-2a209421666d jinja: 'What is the Chinese translation of: {{translation["en"]}} ||| {{translation["zh"]}}' @@ -184,24 +184,39 @@ templates: original_task: true name: !!python/tuple - gpt-3-en-zh-target - reference: '' - 8cbf3dde-9545-4863-aabb-3807f9544c0c: !Template + reference: GPT-3 paper + 744128d8-5265-4dfa-b492-38bee231db22: !Template answer_choices: null - id: 8cbf3dde-9545-4863-aabb-3807f9544c0c - jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation - is: ||| {{translation["zh"]}}' + id: 744128d8-5265-4dfa-b492-38bee231db22 + jinja: 'What is the English translation of : {{translation["zh"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-target + - gpt-3-zh-en-target + reference: GPT-3 paper + 75b55155-1d0b-4d7d-918f-7cff6297f343: !Template + answer_choices: null + id: 75b55155-1d0b-4d7d-918f-7cff6297f343 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_zh-en-source+target reference: '' - 9616be65-d1e1-462a-a27c-05e420914f52: !Template + 81879641-6d30-4a85-8cc8-ad5249564c13: !Template answer_choices: null - id: 9616be65-d1e1-462a-a27c-05e420914f52 - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: 81879641-6d30-4a85-8cc8-ad5249564c13 + jinja: '{{translation["en"]}} = Chinese: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -210,41 +225,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target - reference: '' - a9feb6c0-a8ac-4f9d-aa1b-971c146925ed: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 83f53c84-94e5-4a09-a9e1-910b94ac3670: !Template answer_choices: null - id: a9feb6c0-a8ac-4f9d-aa1b-971c146925ed - jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + id: 83f53c84-94e5-4a09-a9e1-910b94ac3670 + jinja: ' {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-src+target + - translate_as_en-zh-target reference: '' - ad0b11c7-8fe8-4567-8046-1e817a57988a: !Template + 85af687a-da39-48c9-9783-8b7304bae3ea: !Template answer_choices: null - id: ad0b11c7-8fe8-4567-8046-1e817a57988a - jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation - is: + id: 85af687a-da39-48c9-9783-8b7304bae3ea + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-target + - version-zh-en-source+target reference: '' - b26b8e43-2e1a-4020-8ebc-7f5e8ec959c8: !Template + 952df965-758a-4f8b-ae08-2ccbd12b66c0: !Template answer_choices: null - id: b26b8e43-2e1a-4020-8ebc-7f5e8ec959c8 - jinja: 'Translate Chinese into English: {{translation["zh"]}} + id: 952df965-758a-4f8b-ae08-2ccbd12b66c0 + jinja: 'Translate into English: {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -253,63 +268,104 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e3cf11c0-0142-4b35-a518-1df52fc9fa59: !Template + - translate-zh-en-target + reference: '' + 9c27aeb5-b36f-4df9-81b3-c231d7f2a158: !Template answer_choices: null - id: e3cf11c0-0142-4b35-a518-1df52fc9fa59 - jinja: 'What is the English translation of : {{translation["zh"]}} + id: 9c27aeb5-b36f-4df9-81b3-c231d7f2a158 + jinja: 'English: {{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-target - reference: '' - e622d282-3b67-4a8e-a212-83f128bba643: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + a85d88b4-d190-4d86-8f35-685b54c32af0: !Template answer_choices: null - id: e622d282-3b67-4a8e-a212-83f128bba643 - jinja: 'Translate into English: {{translation["zh"]}} + id: a85d88b4-d190-4d86-8f35-685b54c32af0 + jinja: '{{translation["zh"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-target + - translate_as_zh-en-target reference: '' - f325cf8b-28ab-4ce6-bafd-f7c2fc05a741: !Template + b11cc24d-867b-4e0a-a19b-e7e1b518efaf: !Template answer_choices: null - id: f325cf8b-28ab-4ce6-bafd-f7c2fc05a741 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Chinese translation is: + id: b11cc24d-867b-4e0a-a19b-e7e1b518efaf + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-zh-en-source+target + reference: '' + c12af803-17f0-4add-8cbe-49ebbadcbb90: !Template + answer_choices: null + id: c12af803-17f0-4add-8cbe-49ebbadcbb90 + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-source+target + - a_good_translation-zh-en-target reference: '' - f4e0e33d-468d-4bb3-8301-84edc1e579b6: !Template + eacc82a6-ad5d-42c0-a1a4-ff276751d713: !Template answer_choices: null - id: f4e0e33d-468d-4bb3-8301-84edc1e579b6 - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: eacc82a6-ad5d-42c0-a1a4-ff276751d713 + jinja: 'Chinese: {{translation["zh"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f5315dac-8ab1-44e5-9fc7-3d84db8f69cf: !Template + answer_choices: null + id: f5315dac-8ab1-44e5-9fc7-3d84db8f69cf + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - gpt-3-zh-en-source+target + reference: GPT-3 paper + f77ba2f6-01e8-46f2-aac7-b99af59dfb94: !Template + answer_choices: null + id: f77ba2f6-01e8-46f2-aac7-b99af59dfb94 + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: + + ||| {{translation["zh"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-zh-target reference: '' From 443a755fa66afa46ca8ba9652db24a4cbf154530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Yvon?= Date: Wed, 29 Jun 2022 15:29:55 +0200 Subject: [PATCH 14/15] Small change to the translate-* family of prompts - now Translate this from X into Y --- .../templates/wmt14/cs-en/templates.yaml | 284 ++++++++-------- .../templates/wmt14/de-en/templates.yaml | 284 ++++++++-------- .../templates/wmt14/fr-en/templates.yaml | 8 +- .../templates/wmt14/hi-en/templates.yaml | 298 ++++++++--------- .../templates/wmt14/ru-en/templates.yaml | 296 ++++++++--------- .../templates/wmt15/cs-en/templates.yaml | 294 ++++++++--------- .../templates/wmt15/de-en/templates.yaml | 288 ++++++++--------- .../templates/wmt15/fi-en/templates.yaml | 294 ++++++++--------- .../templates/wmt15/fr-en/templates.yaml | 264 +++++++-------- .../templates/wmt15/ru-en/templates.yaml | 282 ++++++++-------- .../templates/wmt16/cs-en/templates.yaml | 284 ++++++++-------- .../templates/wmt16/de-en/templates.yaml | 288 ++++++++--------- .../templates/wmt16/fi-en/templates.yaml | 290 ++++++++--------- .../templates/wmt16/ro-en/templates.yaml | 284 ++++++++-------- .../templates/wmt16/ru-en/templates.yaml | 274 ++++++++-------- .../templates/wmt16/tr-en/templates.yaml | 296 ++++++++--------- .../templates/wmt17/cs-en/templates.yaml | 290 ++++++++--------- .../templates/wmt17/de-en/templates.yaml | 294 ++++++++--------- .../templates/wmt17/fi-en/templates.yaml | 284 ++++++++-------- .../templates/wmt17/lv-en/templates.yaml | 300 ++++++++--------- .../templates/wmt17/ru-en/templates.yaml | 280 ++++++++-------- .../templates/wmt17/tr-en/templates.yaml | 296 ++++++++--------- .../templates/wmt17/zh-en/templates.yaml | 294 ++++++++--------- .../templates/wmt18/cs-en/templates.yaml | 284 ++++++++-------- .../templates/wmt18/de-en/templates.yaml | 304 +++++++++--------- .../templates/wmt18/et-en/templates.yaml | 294 ++++++++--------- .../templates/wmt18/fi-en/templates.yaml | 292 ++++++++--------- .../templates/wmt18/ru-en/templates.yaml | 304 +++++++++--------- .../templates/wmt18/tr-en/templates.yaml | 298 ++++++++--------- .../templates/wmt18/zh-en/templates.yaml | 294 ++++++++--------- .../templates/wmt19/de-en/templates.yaml | 290 ++++++++--------- .../templates/wmt19/fi-en/templates.yaml | 296 ++++++++--------- .../templates/wmt19/fr-de/templates.yaml | 292 ++++++++--------- .../templates/wmt19/gu-en/templates.yaml | 268 +++++++-------- .../templates/wmt19/kk-en/templates.yaml | 296 ++++++++--------- .../templates/wmt19/lt-en/templates.yaml | 288 ++++++++--------- .../templates/wmt19/ru-en/templates.yaml | 288 ++++++++--------- .../templates/wmt19/zh-en/templates.yaml | 288 ++++++++--------- 38 files changed, 5361 insertions(+), 5361 deletions(-) diff --git a/promptsource/templates/wmt14/cs-en/templates.yaml b/promptsource/templates/wmt14/cs-en/templates.yaml index 5ee5872e7..8f429568a 100644 --- a/promptsource/templates/wmt14/cs-en/templates.yaml +++ b/promptsource/templates/wmt14/cs-en/templates.yaml @@ -1,122 +1,123 @@ dataset: wmt14 subset: cs-en templates: - 1ddc856c-16a7-4a0a-a624-6007fdeab8bd: !Template + 027bdc9b-aed7-4509-94ea-38b138f14ddf: !Template answer_choices: null - id: 1ddc856c-16a7-4a0a-a624-6007fdeab8bd - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 027bdc9b-aed7-4509-94ea-38b138f14ddf + jinja: 'Translate this from Czech into English: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target - reference: '' - 325f58a2-55ca-4486-9f31-6791fc5e6641: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 16da9664-30e5-4acc-aff6-21658158cf85: !Template answer_choices: null - id: 325f58a2-55ca-4486-9f31-6791fc5e6641 - jinja: '{{translation["cs"]}} = English: + id: 16da9664-30e5-4acc-aff6-21658158cf85 + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 39a01670-57d3-482c-b53f-eeebcc09cb1d: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 3b3ca652-8414-4442-b55f-c2fc7630b071: !Template answer_choices: null - id: 39a01670-57d3-482c-b53f-eeebcc09cb1d - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 3b3ca652-8414-4442-b55f-c2fc7630b071 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - a_good_translation-en-cs-source+target reference: '' - 4558ada7-1530-4e1b-a8cc-a20f2df5b9da: !Template + 4266e99a-41fa-4c2d-8bbc-91317f42d9ff: !Template answer_choices: null - id: 4558ada7-1530-4e1b-a8cc-a20f2df5b9da - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 4266e99a-41fa-4c2d-8bbc-91317f42d9ff + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target - reference: '' - 5161144a-47c3-4ff5-8f4c-266072f51787: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + 46621847-e018-43fe-b8d1-af439e130254: !Template answer_choices: null - id: 5161144a-47c3-4ff5-8f4c-266072f51787 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 46621847-e018-43fe-b8d1-af439e130254 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-source+target - reference: GPT-3 paper - 560c6739-f946-410a-bd78-f9bc6062a698: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + 4c52dc6a-f16c-4f3c-99d0-274b4162a616: !Template answer_choices: null - id: 560c6739-f946-410a-bd78-f9bc6062a698 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: 4c52dc6a-f16c-4f3c-99d0-274b4162a616 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 56c643ec-d2d6-4b0f-9e9f-68c7e1fc5b2a: !Template + - version-en-cs-target + reference: '' + 5b87693b-ed41-4794-bb3e-b9bf7ada2689: !Template answer_choices: null - id: 56c643ec-d2d6-4b0f-9e9f-68c7e1fc5b2a - jinja: '{{translation["cs"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 5b87693b-ed41-4794-bb3e-b9bf7ada2689 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - a_good_translation-en-cs-target reference: '' - 56cc3bed-3382-4644-93b6-a650315e2c27: !Template + 5e476fb7-5e6e-4732-a084-88e361a82de0: !Template answer_choices: null - id: 56cc3bed-3382-4644-93b6-a650315e2c27 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: 5e476fb7-5e6e-4732-a084-88e361a82de0 + jinja: '{{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: GPT-3 paper - 5f9590ab-e3bf-446e-a815-9134d15d7a04: !Template + - translate_as_cs-en-target + reference: '' + 744bab4b-44d1-4368-a98d-4a403c1a7e0e: !Template answer_choices: null - id: 5f9590ab-e3bf-446e-a815-9134d15d7a04 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: 744bab4b-44d1-4368-a98d-4a403c1a7e0e + jinja: 'Translate this into English: {{translation["cs"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -125,112 +126,111 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target - reference: GPT-3 paper - 6799e0cd-f0d1-4777-8ce8-da82ed2e8dc7: !Template + - translate-cs-en-target + reference: '' + 7fd9d8d7-0c72-42da-a029-ef4d9a9e5f72: !Template answer_choices: null - id: 6799e0cd-f0d1-4777-8ce8-da82ed2e8dc7 - jinja: 'Translate into English: {{translation["cs"]}} - - ||| {{translation["en"]}}' + id: 7fd9d8d7-0c72-42da-a029-ef4d9a9e5f72 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - a_good_translation-cs-en-source+target reference: '' - 79432166-2192-45dc-a3aa-fa75be4456fe: !Template + 809cc7a3-189b-4018-8f81-1e65c2f42271: !Template answer_choices: null - id: 79432166-2192-45dc-a3aa-fa75be4456fe - jinja: 'Translate into Czech: {{translation["en"]}} + id: 809cc7a3-189b-4018-8f81-1e65c2f42271 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target + - a_good_translation-cs-en-target reference: '' - 85cb5985-5cd9-4c7e-b15d-1eb3fd887c23: !Template + 9023b083-ccf1-42a4-9589-5fffd3562649: !Template answer_choices: null - id: 85cb5985-5cd9-4c7e-b15d-1eb3fd887c23 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Czech translation is: + id: 9023b083-ccf1-42a4-9589-5fffd3562649 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-source+target + - version-cs-en-source+target reference: '' - 8d06286c-0584-4548-b2b6-fe5b4dabae9a: !Template + 92e3a4a7-e7c2-4ea1-abfb-be63069d13fa: !Template answer_choices: null - id: 8d06286c-0584-4548-b2b6-fe5b4dabae9a - jinja: 'How do you say {{translation["cs"]}} in English? + id: 92e3a4a7-e7c2-4ea1-abfb-be63069d13fa + jinja: 'Translate this into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - translate-en-cs-target reference: '' - 93853dba-752f-4bdc-a572-9dd2470c5ee2: !Template + a63daae0-77ba-4790-a89d-2428193fbcdc: !Template answer_choices: null - id: 93853dba-752f-4bdc-a572-9dd2470c5ee2 - jinja: 'How do you say {{translation["en"]}} in Czech? + id: a63daae0-77ba-4790-a89d-2428193fbcdc + jinja: 'What is the English translation of : {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target - reference: '' - ae00bea5-318a-496c-a013-72311dac8418: !Template + - gpt-3-cs-en-target + reference: GPT-3 paper + a63fd691-4f07-41da-a98c-a934e866e74e: !Template answer_choices: null - id: ae00bea5-318a-496c-a013-72311dac8418 - jinja: ' {{translation["en"]}} translates into Czech as: + id: a63fd691-4f07-41da-a98c-a934e866e74e + jinja: '{{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target - reference: '' - b0fc060e-9c17-4b89-b931-af0027a8fba1: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ab264706-636e-4bd0-a762-860029b54f45: !Template answer_choices: null - id: b0fc060e-9c17-4b89-b931-af0027a8fba1 - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version - should say: + id: ab264706-636e-4bd0-a762-860029b54f45 + jinja: 'Translate this from English into Czech: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target + - translate-en-cs-source+starget reference: '' - bde02500-7413-46a8-aa2f-438bfa77275d: !Template + ac25c74b-1a81-409e-9c4f-bdc94912721c: !Template answer_choices: null - id: bde02500-7413-46a8-aa2f-438bfa77275d - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: ac25c74b-1a81-409e-9c4f-bdc94912721c + jinja: ' {{translation["en"]}} translates into Czech as: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -239,25 +239,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - translate_as_en-cs-target reference: '' - c45fbb7a-dabf-4990-a63e-cf2543050511: !Template + b42858a8-35d7-4172-ace4-80df1f6a94e0: !Template answer_choices: null - id: c45fbb7a-dabf-4990-a63e-cf2543050511 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: b42858a8-35d7-4172-ace4-80df1f6a94e0 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - version-en-cs-source+target reference: '' - cc19db28-9a53-4ffc-9f98-9e399cba8b33: !Template + c4629789-74ce-4378-8d03-f75b2327ad77: !Template answer_choices: null - id: cc19db28-9a53-4ffc-9f98-9e399cba8b33 - jinja: '{{translation["en"]}} = Czech: + id: c4629789-74ce-4378-8d03-f75b2327ad77 + jinja: 'English: {{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -266,13 +268,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d01581c4-3bfe-4290-9ce6-04f354660f1b: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + cdce6dd9-52ad-42c3-a691-8efb13ab535f: !Template answer_choices: null - id: d01581c4-3bfe-4290-9ce6-04f354660f1b - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: cdce6dd9-52ad-42c3-a691-8efb13ab535f + jinja: 'Czech: {{translation["cs"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -281,26 +282,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - translate_as_cs-en-source+target reference: '' - d13d8c55-6569-48ee-a12e-4a1ed7351652: !Template + dcefc4dc-560d-4bf5-8623-c99d535232b0: !Template answer_choices: null - id: d13d8c55-6569-48ee-a12e-4a1ed7351652 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: dcefc4dc-560d-4bf5-8623-c99d535232b0 + jinja: 'How do you say {{translation["cs"]}} in English? - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - how_to_say-cs-en-target reference: '' - e2ff2252-88e6-423d-a992-34f5fcf2f36e: !Template + e13ee973-fce4-4d42-8ec5-b871c20eaf39: !Template answer_choices: null - id: e2ff2252-88e6-423d-a992-34f5fcf2f36e + id: e13ee973-fce4-4d42-8ec5-b871c20eaf39 jinja: 'Czech: {{translation["cs"]}} = English: ||| {{translation["en"]}}' @@ -312,53 +312,53 @@ templates: name: !!python/tuple - xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - e87df111-e881-4887-b3b2-ba2fd778b1ce: !Template + ee298b4c-8165-44cc-81b6-1fb18f4e9e02: !Template answer_choices: null - id: e87df111-e881-4887-b3b2-ba2fd778b1ce - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: ee298b4c-8165-44cc-81b6-1fb18f4e9e02 + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - translate_as_en-cs-source+target reference: '' - f2b5cc24-d38e-4085-9348-fa2a77b580d9: !Template + f7af3145-3444-4183-921b-a549fef5e1b3: !Template answer_choices: null - id: f2b5cc24-d38e-4085-9348-fa2a77b580d9 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: f7af3145-3444-4183-921b-a549fef5e1b3 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-source+target - reference: GPT 3 paper - f6b141d3-36ba-4cbb-a0f5-2d03ecc19a66: !Template + - version-cs-en-target + reference: '' + f7d630ca-1791-4aaf-97b5-7c15e5eba296: !Template answer_choices: null - id: f6b141d3-36ba-4cbb-a0f5-2d03ecc19a66 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: f7d630ca-1791-4aaf-97b5-7c15e5eba296 + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-source+starget + - how_to_say-en-cs-target reference: '' - faa45418-ed53-461b-abfb-267790bfea6a: !Template + f83cb5c3-7989-49d9-a1cb-6fdc16b445e3: !Template answer_choices: null - id: faa45418-ed53-461b-abfb-267790bfea6a - jinja: 'English: {{translation["en"]}} = Czech: + id: f83cb5c3-7989-49d9-a1cb-6fdc16b445e3 + jinja: '{{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -367,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml index f34f9cc99..389dc09c0 100644 --- a/promptsource/templates/wmt14/de-en/templates.yaml +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -1,108 +1,110 @@ dataset: wmt14 subset: de-en templates: - 018980f2-de9d-45e5-b2b6-29a68bb4cc12: !Template + 032ecc70-2a7b-4ba9-8adf-c722741723a3: !Template answer_choices: null - id: 018980f2-de9d-45e5-b2b6-29a68bb4cc12 - jinja: '{{translation["de"]}} translates into English as: + id: 032ecc70-2a7b-4ba9-8adf-c722741723a3 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - version-de-en-target reference: '' - 02c6672a-d29b-4604-b60d-74e04317c362: !Template + 1bd45cc7-cfe0-4ad9-a157-e585b7931afe: !Template answer_choices: null - id: 02c6672a-d29b-4604-b60d-74e04317c362 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 1bd45cc7-cfe0-4ad9-a157-e585b7931afe + jinja: 'English: {{translation["en"]}} = German: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target - reference: '' - 03158c3a-0709-4489-909d-2f2c44b9b7c6: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 1d5e9684-1964-45f9-90fd-5e6bde7d22f4: !Template answer_choices: null - id: 03158c3a-0709-4489-909d-2f2c44b9b7c6 - jinja: 'English: {{translation["en"]}} = German: + id: 1d5e9684-1964-45f9-90fd-5e6bde7d22f4 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 0bfa68cc-2054-4d31-85df-787b24b0c0c2: !Template + - version-de-en-source+target + reference: '' + 1dcd49d2-0b77-4a6e-907b-c39453282bfe: !Template answer_choices: null - id: 0bfa68cc-2054-4d31-85df-787b24b0c0c2 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 1dcd49d2-0b77-4a6e-907b-c39453282bfe + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target - reference: GPT-3 paper - 12459bda-43d1-4b92-9128-206b49eb4230: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 27a87c3b-1516-4a1c-b834-141b11b85f20: !Template answer_choices: null - id: 12459bda-43d1-4b92-9128-206b49eb4230 - jinja: 'Translate German into English: {{translation["de"]}} + id: 27a87c3b-1516-4a1c-b834-141b11b85f20 + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1aed34d3-71c0-4478-acf9-905ac7f68cf9: !Template + - translate_as_en-de-source+target + reference: '' + 2f928e28-04d8-4538-bea4-545a072173ed: !Template answer_choices: null - id: 1aed34d3-71c0-4478-acf9-905ac7f68cf9 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 2f928e28-04d8-4538-bea4-545a072173ed + jinja: 'Translate this from English into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - translate-en-de-source+starget reference: '' - 22d04cb6-3603-4d49-96f5-227410929d9c: !Template + 3210d998-8cd4-420c-9ea4-3fa3c67aa5f7: !Template answer_choices: null - id: 22d04cb6-3603-4d49-96f5-227410929d9c - jinja: 'English: {{translation["en"]}} translates into German as: + id: 3210d998-8cd4-420c-9ea4-3fa3c67aa5f7 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target - reference: '' - 3432fbc9-d1c2-437f-97c9-23c048e39197: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 36d02443-77b4-4f6b-9220-838f996b2f28: !Template answer_choices: null - id: 3432fbc9-d1c2-437f-97c9-23c048e39197 - jinja: 'How do you say {{translation["de"]}} in English? + id: 36d02443-77b4-4f6b-9220-838f996b2f28 + jinja: '{{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -111,41 +113,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target - reference: '' - 41a8dad3-0d97-4c91-a232-749f28e27697: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 42dc9689-b971-4641-b6e3-97f55e6da9b7: !Template answer_choices: null - id: 41a8dad3-0d97-4c91-a232-749f28e27697 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 42dc9689-b971-4641-b6e3-97f55e6da9b7 + jinja: 'Translate this from German into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target - reference: '' - 572f7058-68a2-4c60-aa38-56e4da45b6e6: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 44ecf43e-f4ae-4718-a730-945e86777917: !Template answer_choices: null - id: 572f7058-68a2-4c60-aa38-56e4da45b6e6 - jinja: ' {{translation["en"]}} translates into German as: + id: 44ecf43e-f4ae-4718-a730-945e86777917 + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - translate_as_de-en-target reference: '' - 5cc7fb46-364b-4494-9557-40e579258f0f: !Template + 54d4f276-b5f4-4895-8e62-fc23362670b0: !Template answer_choices: null - id: 5cc7fb46-364b-4494-9557-40e579258f0f - jinja: '{{translation["en"]}} = German: + id: 54d4f276-b5f4-4895-8e62-fc23362670b0 + jinja: ' {{translation["en"]}} translates into German as: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -154,26 +155,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 6088a5e1-a753-4d55-a977-6b1fbb3d5ebc: !Template + - translate_as_en-de-target + reference: '' + 6e69d25e-6209-41d3-b2ee-6d18c861f34f: !Template answer_choices: null - id: 6088a5e1-a753-4d55-a977-6b1fbb3d5ebc - jinja: 'How do you say {{translation["en"]}} in German? + id: 6e69d25e-6209-41d3-b2ee-6d18c861f34f + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target + - how_to_say-de-en-target reference: '' - 68222c7d-2854-4289-9207-33c072b9c382: !Template + 719762e7-0928-40fc-8c75-274c99586135: !Template answer_choices: null - id: 68222c7d-2854-4289-9207-33c072b9c382 - jinja: 'Translate into German: {{translation["en"]}} + id: 719762e7-0928-40fc-8c75-274c99586135 + jinja: 'Translate this into German: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -184,51 +185,55 @@ templates: name: !!python/tuple - translate-en-de-target reference: '' - 6e98a0d2-595f-4c7a-a253-3ec0325846b5: !Template + 886c41e7-cbc7-405f-8d8c-f6d3f11ee473: !Template answer_choices: null - id: 6e98a0d2-595f-4c7a-a253-3ec0325846b5 - jinja: 'Translate English into German: {{translation["en"]}} + id: 886c41e7-cbc7-405f-8d8c-f6d3f11ee473 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget + - version-en-de-target reference: '' - 7b8e23c3-a170-4ac9-9fd2-5c8142144f58: !Template + 8bc4714f-f068-4d48-a661-0666c2f66193: !Template answer_choices: null - id: 7b8e23c3-a170-4ac9-9fd2-5c8142144f58 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 8bc4714f-f068-4d48-a661-0666c2f66193 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - a_good_translation-en-de-source+target reference: '' - 7e21750f-6aa2-4893-9b1a-5f6b30dc9b4d: !Template + 8ec97fe5-268c-4a3f-a173-1bbaa0d2104f: !Template answer_choices: null - id: 7e21750f-6aa2-4893-9b1a-5f6b30dc9b4d - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 8ec97fe5-268c-4a3f-a173-1bbaa0d2104f + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-source+target - reference: GPT 3 paper - 87400fa8-12d7-4ca2-b843-e45c9c9bdefc: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 91d21ef8-c0c9-473d-b4e0-ba962664cd90: !Template answer_choices: null - id: 87400fa8-12d7-4ca2-b843-e45c9c9bdefc - jinja: '{{translation["de"]}} = English: + id: 91d21ef8-c0c9-473d-b4e0-ba962664cd90 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -237,12 +242,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9ce7e8f7-c563-44b2-85bf-e3ae36e2ced0: !Template + - a_good_translation-de-en-target + reference: '' + 926fb8b4-d2a5-428c-aee0-fac54586bcc7: !Template answer_choices: null - id: 9ce7e8f7-c563-44b2-85bf-e3ae36e2ced0 - jinja: 'Translate into English: {{translation["de"]}} + id: 926fb8b4-d2a5-428c-aee0-fac54586bcc7 + jinja: 'Translate this into English: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -253,39 +258,37 @@ templates: name: !!python/tuple - translate-de-en-target reference: '' - a77aebb0-b708-466a-8866-138fc1813b17: !Template + a7f186b5-3ec2-46b6-824a-46a68f102514: !Template answer_choices: null - id: a77aebb0-b708-466a-8866-138fc1813b17 - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: a7f186b5-3ec2-46b6-824a-46a68f102514 + jinja: '{{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target - reference: '' - b306075f-da0c-4d28-98eb-20d4472987dc: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ae772d49-5a93-4ffd-b1a7-9e2034a23509: !Template answer_choices: null - id: b306075f-da0c-4d28-98eb-20d4472987dc - jinja: 'What is the English translation of : {{translation["de"]}} - - ||| {{translation["en"]}}' + id: ae772d49-5a93-4ffd-b1a7-9e2034a23509 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: GPT-3 paper - c468702f-5ba2-49e9-a1c7-bddc3367a4be: !Template + - a_good_translation-de-en-source+target + reference: '' + aeb84e76-e929-4dd6-8002-69511c2e9fb6: !Template answer_choices: null - id: c468702f-5ba2-49e9-a1c7-bddc3367a4be - jinja: 'If the original version says: {{translation["en"]}}; then the German version + id: aeb84e76-e929-4dd6-8002-69511c2e9fb6 + jinja: 'If the English version says: {{translation["en"]}}; then the German version should say: ||| {{translation["de"]}}' @@ -295,40 +298,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - version-en-de-source+target reference: '' - cec39bc6-0742-4a8c-9e27-dcecfde865cb: !Template + b57234a1-0d63-40b3-909c-14c8a8896c7c: !Template answer_choices: null - id: cec39bc6-0742-4a8c-9e27-dcecfde865cb - jinja: 'German: {{translation["de"]}} = English: - - ||| {{translation["en"]}}' + id: b57234a1-0d63-40b3-909c-14c8a8896c7c + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d3db7a9c-7021-49eb-a7ec-df93b3c4c731: !Template + - a_good_translation-en-de-target + reference: '' + c6d34117-0d80-4e19-a669-481293148bab: !Template answer_choices: null - id: d3db7a9c-7021-49eb-a7ec-df93b3c4c731 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: c6d34117-0d80-4e19-a669-481293148bab + jinja: 'What is the German translation of: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - dfb9e96b-436f-4c19-a746-ae5f3f1145c8: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + d04ece63-a018-4f93-8b2b-e8d48ed239e8: !Template answer_choices: null - id: dfb9e96b-436f-4c19-a746-ae5f3f1145c8 + id: d04ece63-a018-4f93-8b2b-e8d48ed239e8 jinja: 'German: {{translation["de"]}} translates into English as: ||| {{translation["en"]}}' @@ -340,32 +341,31 @@ templates: name: !!python/tuple - translate_as_de-en-source+target reference: '' - e4601e49-0d41-4f89-af5e-e5866a618f75: !Template + dc6bcb43-5b88-4b16-9863-51bbad17ddd6: !Template answer_choices: null - id: e4601e49-0d41-4f89-af5e-e5866a618f75 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: dc6bcb43-5b88-4b16-9863-51bbad17ddd6 + jinja: 'What is the English translation of : {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target - reference: '' - fb86ef86-ea62-48b2-bb6f-d63d19f86510: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + eaee5677-f889-4bfd-8fbb-ad976b917506: !Template answer_choices: null - id: fb86ef86-ea62-48b2-bb6f-d63d19f86510 - jinja: 'What is the German translation of: {{translation["en"]}} + id: eaee5677-f889-4bfd-8fbb-ad976b917506 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: GPT-3 paper + - how_to_say-en-de-target + reference: '' diff --git a/promptsource/templates/wmt14/fr-en/templates.yaml b/promptsource/templates/wmt14/fr-en/templates.yaml index f2d6a2e5a..d29c96bc0 100644 --- a/promptsource/templates/wmt14/fr-en/templates.yaml +++ b/promptsource/templates/wmt14/fr-en/templates.yaml @@ -32,7 +32,7 @@ templates: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb: !Template answer_choices: null id: 1fbf99d9-980a-4bf8-b30e-dcf9e3ad5feb - jinja: 'Translate French into English: {{translation["fr"]}} + jinja: 'Translate this from French into English: {{translation["fr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -84,7 +84,7 @@ templates: 41e7647e-2b9f-4d86-8987-5abe70000362: !Template answer_choices: null id: 41e7647e-2b9f-4d86-8987-5abe70000362 - jinja: 'Translate into English: {{translation["fr"]}} + jinja: 'Translate this into English: {{translation["fr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -147,7 +147,7 @@ templates: 527d2074-74fa-47ec-802e-7373728230df: !Template answer_choices: null id: 527d2074-74fa-47ec-802e-7373728230df - jinja: 'Translate into French: {{translation["en"]}} + jinja: 'Translate this into French: {{translation["en"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -160,7 +160,7 @@ templates: 5af8c873-7176-46b8-a31a-2b2d393c6c68: !Template answer_choices: null id: 5af8c873-7176-46b8-a31a-2b2d393c6c68 - jinja: 'Translate English into French: {{translation["en"]}} + jinja: 'Translate this from English into French: {{translation["en"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata diff --git a/promptsource/templates/wmt14/hi-en/templates.yaml b/promptsource/templates/wmt14/hi-en/templates.yaml index c7475686f..cf9bb90d8 100644 --- a/promptsource/templates/wmt14/hi-en/templates.yaml +++ b/promptsource/templates/wmt14/hi-en/templates.yaml @@ -1,123 +1,108 @@ dataset: wmt14 subset: hi-en templates: - 0597ecbe-44c0-40ad-b22c-e52ac7d28f03: !Template + 02ec7175-a97c-4c0f-982f-1cc8c4c050d1: !Template answer_choices: null - id: 0597ecbe-44c0-40ad-b22c-e52ac7d28f03 - jinja: '{{translation["en"]}} = Hindi: - - ||| {{translation["hi"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - xglm-en-hi-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 32166763-eb01-41a1-8793-51125460f5cc: !Template - answer_choices: null - id: 32166763-eb01-41a1-8793-51125460f5cc - jinja: 'How do you say {{translation["hi"]}} in English? + id: 02ec7175-a97c-4c0f-982f-1cc8c4c050d1 + jinja: 'If the original version says: {{translation["hi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-hi-en-target + - version-hi-en-target reference: '' - 3be84903-3560-4481-b35c-486daefecd67: !Template + 03664fac-67ef-414d-8e4a-504ad4d7a8a0: !Template answer_choices: null - id: 3be84903-3560-4481-b35c-486daefecd67 - jinja: 'Given the following passage: {{translation["en"]}} , a good Hindi translation - is: ||| {{translation["hi"]}}' + id: 03664fac-67ef-414d-8e4a-504ad4d7a8a0 + jinja: 'Given the following source text in Hindi: {{translation["hi"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-hi-target + - a_good_translation-hi-en-source+target reference: '' - 3d857ce6-7f1f-440f-802c-72e52fbf3bdd: !Template + 1c4fc122-d6e1-41f5-91e8-1aacbbfb3549: !Template answer_choices: null - id: 3d857ce6-7f1f-440f-802c-72e52fbf3bdd - jinja: 'English: {{translation["en"]}} translates into Hindi as: + id: 1c4fc122-d6e1-41f5-91e8-1aacbbfb3549 + jinja: 'What is the English translation of the Hindi sentence: {{translation["hi"]}} - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-hi-source+target - reference: '' - 4094f562-64c5-48fe-b413-6d8fde1084ee: !Template + - gpt-3-hi-en-source+target + reference: GPT-3 paper + 1dcf0a6e-1463-4c07-8431-95ba24273257: !Template answer_choices: null - id: 4094f562-64c5-48fe-b413-6d8fde1084ee - jinja: 'If the original version says: {{translation["hi"]}}; then the English - version should say: + id: 1dcf0a6e-1463-4c07-8431-95ba24273257 + jinja: 'How do you say {{translation["en"]}} in Hindi? - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-hi-en-target + - how_to_say-en-hi-target reference: '' - 436f9f1c-916a-40ae-9b75-3db5d3a711f3: !Template + 25daf37e-e684-419e-a250-bdeeb82a7df6: !Template answer_choices: null - id: 436f9f1c-916a-40ae-9b75-3db5d3a711f3 - jinja: '{{translation["hi"]}} = English: + id: 25daf37e-e684-419e-a250-bdeeb82a7df6 + jinja: '{{translation["en"]}} = Hindi: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-hi-en-target + - xglm-en-hi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4c21064e-83e2-41b3-a0db-45373e12fa26: !Template + 267ca01b-599f-46d6-a85d-ce1915382c18: !Template answer_choices: null - id: 4c21064e-83e2-41b3-a0db-45373e12fa26 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Hindi translation is: + id: 267ca01b-599f-46d6-a85d-ce1915382c18 + jinja: 'Translate this from Hindi into English: {{translation["hi"]}} - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-hi-source+target - reference: '' - 58e9f511-1be7-4255-abed-7b97d6197036: !Template + - translate-hi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 2fd2a8ba-ea2a-4686-bee5-2f49fb31ebca: !Template answer_choices: null - id: 58e9f511-1be7-4255-abed-7b97d6197036 - jinja: 'Hindi: {{translation["hi"]}} translates into English as: + id: 2fd2a8ba-ea2a-4686-bee5-2f49fb31ebca + jinja: 'Translate this from English into Hindi: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_hi-en-source+target + - translate-en-hi-source+starget reference: '' - 7cf403e3-5b6c-44fc-9046-57bfbc92e05e: !Template + 40d4d47a-fd09-48af-88d2-ef3ee7a377d6: !Template answer_choices: null - id: 7cf403e3-5b6c-44fc-9046-57bfbc92e05e - jinja: 'Translate Hindi into English: {{translation["hi"]}} + id: 40d4d47a-fd09-48af-88d2-ef3ee7a377d6 + jinja: 'Translate this into English: {{translation["hi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -126,54 +111,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-hi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 819805c2-ca6b-4a2f-8c67-db22b92210f4: !Template - answer_choices: null - id: 819805c2-ca6b-4a2f-8c67-db22b92210f4 - jinja: 'How do you say {{translation["en"]}} in Hindi? - - ||| {{translation["hi"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - how_to_say-en-hi-target + - translate-hi-en-target reference: '' - 914321b1-e420-426e-b491-b1d9869d50d6: !Template + 42379c42-04c5-4ea9-99ca-f43f1b1cfc1b: !Template answer_choices: null - id: 914321b1-e420-426e-b491-b1d9869d50d6 - jinja: 'What is the English translation of the Hindi sentence: {{translation["hi"]}} - - ||| {{translation["en"]}}' + id: 42379c42-04c5-4ea9-99ca-f43f1b1cfc1b + jinja: 'Given the following passage: {{translation["en"]}} , a good Hindi translation + is: ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-hi-en-source+target - reference: GPT-3 paper - 94fbd505-da03-4e7d-a16f-2a48ba523e2a: !Template + - a_good_translation-en-hi-target + reference: '' + 42dd0622-3e2a-4fa0-b3fc-6087225503c9: !Template answer_choices: null - id: 94fbd505-da03-4e7d-a16f-2a48ba523e2a - jinja: 'Translate English into Hindi: {{translation["en"]}} + id: 42dd0622-3e2a-4fa0-b3fc-6087225503c9 + jinja: 'Hindi: {{translation["hi"]}} = English: - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-hi-source+starget - reference: '' - 9eae8a37-27be-42de-82ed-b32998f719ce: !Template + - xglm-hi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5149f4af-3272-42dd-a4d5-d3cd83927655: !Template answer_choices: null - id: 9eae8a37-27be-42de-82ed-b32998f719ce - jinja: '{{translation["hi"]}} translates into English as: + id: 5149f4af-3272-42dd-a4d5-d3cd83927655 + jinja: 'How do you say {{translation["hi"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -182,27 +152,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_hi-en-target + - how_to_say-hi-en-target reference: '' - a20352ec-8ef1-42ed-8827-001374fc1ee5: !Template + 54a82bf0-4765-4e10-9311-1f07f6c22799: !Template answer_choices: null - id: a20352ec-8ef1-42ed-8827-001374fc1ee5 - jinja: 'If the Hindi version says: {{translation["hi"]}}; then the English version - should say: + id: 54a82bf0-4765-4e10-9311-1f07f6c22799 + jinja: 'English: {{translation["en"]}} = Hindi: - ||| {{translation["en"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-hi-en-source+target - reference: '' - af072fb1-3955-4b6f-8489-ed2d0c9c61cd: !Template + - xglm-en-hi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 678d458a-f40c-480c-811f-f8aa51cda71a: !Template answer_choices: null - id: af072fb1-3955-4b6f-8489-ed2d0c9c61cd - jinja: 'Translate into Hindi: {{translation["en"]}} + id: 678d458a-f40c-480c-811f-f8aa51cda71a + jinja: 'Translate this into Hindi: {{translation["en"]}} ||| {{translation["hi"]}}' metadata: !TemplateMetadata @@ -213,24 +182,24 @@ templates: name: !!python/tuple - translate-en-hi-target reference: '' - b134b1da-2286-4d38-bc3d-1f66c92de1cb: !Template + 8116f3dd-3066-4a47-9a0c-92884d98f6c2: !Template answer_choices: null - id: b134b1da-2286-4d38-bc3d-1f66c92de1cb - jinja: 'What is the English translation of : {{translation["hi"]}} + id: 8116f3dd-3066-4a47-9a0c-92884d98f6c2 + jinja: '{{translation["hi"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-hi-en-target - reference: GPT-3 paper - b91acbc0-81fd-498a-aef5-e61ae66d659b: !Template + - translate_as_hi-en-target + reference: '' + 84b903ab-e138-4975-8507-77be58bc0fb0: !Template answer_choices: null - id: b91acbc0-81fd-498a-aef5-e61ae66d659b - jinja: ' {{translation["en"]}} translates into Hindi as: + id: 84b903ab-e138-4975-8507-77be58bc0fb0 + jinja: 'English: {{translation["en"]}} translates into Hindi as: ||| {{translation["hi"]}}' metadata: !TemplateMetadata @@ -239,12 +208,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-hi-target + - translate_as_en-hi-source+target reference: '' - cc2e0824-a3a8-4ed1-bde2-9b3b893eead3: !Template + 9b430f52-31a1-4b7b-9600-59069a706b2c: !Template answer_choices: null - id: cc2e0824-a3a8-4ed1-bde2-9b3b893eead3 - jinja: 'Hindi: {{translation["hi"]}} = English: + id: 9b430f52-31a1-4b7b-9600-59069a706b2c + jinja: '{{translation["hi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -253,69 +222,68 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-hi-en-source+target + - xglm-hi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - cda44776-363a-4a2d-aceb-07416ff531f1: !Template + 9f9c17e6-25b3-4240-b0d3-61f47b9c2523: !Template answer_choices: null - id: cda44776-363a-4a2d-aceb-07416ff531f1 - jinja: 'Translate into English: {{translation["hi"]}} + id: 9f9c17e6-25b3-4240-b0d3-61f47b9c2523 + jinja: 'Hindi: {{translation["hi"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-hi-en-target + - translate_as_hi-en-source+target reference: '' - e2b4e89e-d1f4-4655-8147-7fa3839a6f0b: !Template + a5d28474-3b52-4723-98e5-1330ca121381: !Template answer_choices: null - id: e2b4e89e-d1f4-4655-8147-7fa3839a6f0b - jinja: 'If the English version says: {{translation["en"]}}; then the Hindi version + id: a5d28474-3b52-4723-98e5-1330ca121381 + jinja: 'If the Hindi version says: {{translation["hi"]}}; then the English version should say: - ||| {{translation["hi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-hi-source+target + - version-hi-en-source+target reference: '' - e5025d4c-85e9-49c9-9c44-f3b3da3b9710: !Template + ab0195bd-4abd-4d9e-8107-afa7a3a6f6fc: !Template answer_choices: null - id: e5025d4c-85e9-49c9-9c44-f3b3da3b9710 - jinja: 'Given the following passage: {{translation["hi"]}} , a good English translation - is: + id: ab0195bd-4abd-4d9e-8107-afa7a3a6f6fc + jinja: 'What is the English translation of : {{translation["hi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-hi-en-target - reference: '' - f146c715-e2b8-4952-aa26-7df96c37e73b: !Template + - gpt-3-hi-en-target + reference: GPT-3 paper + acc9d2cb-e156-4907-a9ae-9b85b714ce19: !Template answer_choices: null - id: f146c715-e2b8-4952-aa26-7df96c37e73b - jinja: 'What is the Hindi translation of: {{translation["en"]}} + id: acc9d2cb-e156-4907-a9ae-9b85b714ce19 + jinja: ' {{translation["en"]}} translates into Hindi as: - ||| {{translation["hi"]}}' + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-hi-target - reference: GPT-3 paper - f72a0b57-1dce-4e87-a43d-0ee0deadcf11: !Template + - translate_as_en-hi-target + reference: '' + b5952cac-9388-4901-98ed-c45cccfed5de: !Template answer_choices: null - id: f72a0b57-1dce-4e87-a43d-0ee0deadcf11 + id: b5952cac-9388-4901-98ed-c45cccfed5de jinja: 'If the original version says: {{translation["en"]}}; then the Hindi version should say: @@ -328,10 +296,11 @@ templates: name: !!python/tuple - version-en-hi-target reference: '' - f96c3c81-02fd-4435-b09e-7a5dc161e612: !Template + d1e354a7-8fa3-415a-9bb7-755e1ae21813: !Template answer_choices: null - id: f96c3c81-02fd-4435-b09e-7a5dc161e612 - jinja: 'English: {{translation["en"]}} = Hindi: + id: d1e354a7-8fa3-415a-9bb7-755e1ae21813 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Hindi translation is: ||| {{translation["hi"]}}' metadata: !TemplateMetadata @@ -340,11 +309,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-hi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - fef51115-c777-4eb6-8eeb-8b71f227b201: !Template + - a_good_translation-en-hi-source+target + reference: '' + d3796afb-95f9-4a69-b053-1340e3b86626: !Template answer_choices: null - id: fef51115-c777-4eb6-8eeb-8b71f227b201 + id: d3796afb-95f9-4a69-b053-1340e3b86626 jinja: 'What is the Hindi translation of the English sentence: {{translation["en"]}} ||| {{translation["hi"]}}' @@ -356,16 +325,47 @@ templates: name: !!python/tuple - gpt-3-en-hi-source+target reference: GPT 3 paper - ff581230-e0a6-45a5-9c87-f5b89872e2da: !Template + eb27a29c-e238-4ebd-a675-456d2102a80e: !Template answer_choices: null - id: ff581230-e0a6-45a5-9c87-f5b89872e2da - jinja: 'Given the following source text in Hindi: {{translation["hi"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: eb27a29c-e238-4ebd-a675-456d2102a80e + jinja: 'What is the Hindi translation of: {{translation["en"]}} + + ||| {{translation["hi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-hi-en-source+target + - gpt-3-en-hi-target + reference: GPT-3 paper + f5e8dd73-6e5c-48a7-b92a-766b5ce9b5e5: !Template + answer_choices: null + id: f5e8dd73-6e5c-48a7-b92a-766b5ce9b5e5 + jinja: 'If the English version says: {{translation["en"]}}; then the Hindi version + should say: + + ||| {{translation["hi"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - version-en-hi-source+target + reference: '' + fbd2d598-80e9-4ce6-b85e-fb269aa82580: !Template + answer_choices: null + id: fbd2d598-80e9-4ce6-b85e-fb269aa82580 + jinja: 'Given the following passage: {{translation["hi"]}} , a good English translation + is: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-hi-en-target reference: '' diff --git a/promptsource/templates/wmt14/ru-en/templates.yaml b/promptsource/templates/wmt14/ru-en/templates.yaml index a67763e62..b35f56109 100644 --- a/promptsource/templates/wmt14/ru-en/templates.yaml +++ b/promptsource/templates/wmt14/ru-en/templates.yaml @@ -1,83 +1,81 @@ dataset: wmt14 subset: ru-en templates: - 073e3350-9315-4a9d-8330-091354036446: !Template + 01a32576-209b-4ce8-b3f3-12b446ff21b0: !Template answer_choices: null - id: 073e3350-9315-4a9d-8330-091354036446 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 01a32576-209b-4ce8-b3f3-12b446ff21b0 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target - reference: '' - 17d195b0-cc2f-4369-a578-6cdeb435289a: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 073879bd-c1ca-429e-89e6-8e8ad608b91b: !Template answer_choices: null - id: 17d195b0-cc2f-4369-a578-6cdeb435289a - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: - - ||| {{translation["en"]}}' + id: 073879bd-c1ca-429e-89e6-8e8ad608b91b + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - a_good_translation-en-ru-target reference: '' - 19f6f139-8914-4d0b-be91-7ce530bca4ca: !Template + 255dcbb5-6d94-4a71-b868-1ada8b050c84: !Template answer_choices: null - id: 19f6f139-8914-4d0b-be91-7ce530bca4ca - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 255dcbb5-6d94-4a71-b868-1ada8b050c84 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - version-ru-en-target reference: '' - 1a198507-ff2a-458a-aa01-c758468cc87d: !Template + 2fa8c0fd-cdf5-471d-96ca-c24768ac0409: !Template answer_choices: null - id: 1a198507-ff2a-458a-aa01-c758468cc87d - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 2fa8c0fd-cdf5-471d-96ca-c24768ac0409 + jinja: 'Translate this from English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1ec62a81-d73d-4987-a874-477c39d0924e: !Template + - translate-en-ru-source+starget + reference: '' + 333bc458-f648-47c5-8bd9-699c901da9c8: !Template answer_choices: null - id: 1ec62a81-d73d-4987-a874-477c39d0924e - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 333bc458-f648-47c5-8bd9-699c901da9c8 + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' - 2156cb8f-f81e-4a26-a1ec-6b8f94c76dcd: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 38d575d3-b90c-4b53-b65c-91fd94196282: !Template answer_choices: null - id: 2156cb8f-f81e-4a26-a1ec-6b8f94c76dcd - jinja: ' {{translation["en"]}} translates into Russian as: + id: 38d575d3-b90c-4b53-b65c-91fd94196282 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -86,12 +84,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - a_good_translation-en-ru-source+target reference: '' - 2299e8ae-d964-4059-8d7b-61dfc5e02d9a: !Template + 3afd655a-42c2-4a47-95ef-193ded37e470: !Template answer_choices: null - id: 2299e8ae-d964-4059-8d7b-61dfc5e02d9a - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 3afd655a-42c2-4a47-95ef-193ded37e470 + jinja: 'English: {{translation["en"]}} = Russian: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -100,82 +98,83 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target - reference: '' - 29874269-2e7d-4dee-82f6-2b64941a808b: !Template + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 4dad78cb-8811-4319-be5b-d8dc9380c0ac: !Template answer_choices: null - id: 29874269-2e7d-4dee-82f6-2b64941a808b - jinja: 'Russian: {{translation["ru"]}} = English: + id: 4dad78cb-8811-4319-be5b-d8dc9380c0ac + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 3162c64d-79f2-4ca8-9b0e-f1b20cd7b66b: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 7b30c457-c1eb-4ac2-8c61-9edfff3aa89e: !Template answer_choices: null - id: 3162c64d-79f2-4ca8-9b0e-f1b20cd7b66b - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 7b30c457-c1eb-4ac2-8c61-9edfff3aa89e + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: GPT-3 paper - 435ca8a6-015d-4e87-920e-c7063de43871: !Template + - translate_as_ru-en-source+target + reference: '' + 7d060eb3-ae66-4479-89b3-666af83e3cbb: !Template answer_choices: null - id: 435ca8a6-015d-4e87-920e-c7063de43871 - jinja: '{{translation["en"]}} = Russian: + id: 7d060eb3-ae66-4479-89b3-666af83e3cbb + jinja: 'Translate this into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 52e615f9-bb28-47c0-93d8-68474bf50e26: !Template + - translate-ru-en-target + reference: '' + 85f57cfb-454e-44b8-9065-a19707060565: !Template answer_choices: null - id: 52e615f9-bb28-47c0-93d8-68474bf50e26 - jinja: 'How do you say {{translation["ru"]}} in English? + id: 85f57cfb-454e-44b8-9065-a19707060565 + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate_as_en-ru-target reference: '' - 576fd9ec-e97e-4077-b533-433dcc9a12d6: !Template + 8aed1935-84a3-423e-945c-3988d59dae43: !Template answer_choices: null - id: 576fd9ec-e97e-4077-b533-433dcc9a12d6 - jinja: 'Translate into Russian: {{translation["en"]}} + id: 8aed1935-84a3-423e-945c-3988d59dae43 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - version-ru-en-source+target reference: '' - 78fca8ed-a720-482e-9a8b-8b45308deda7: !Template + 9306ac01-a84c-4219-aa7c-5b33470597b5: !Template answer_choices: null - id: 78fca8ed-a720-482e-9a8b-8b45308deda7 - jinja: 'English: {{translation["en"]}} = Russian: + id: 9306ac01-a84c-4219-aa7c-5b33470597b5 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -184,55 +183,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 876e475c-7199-4ad9-8a14-dd8a367fadd9: !Template + - translate_as_en-ru-source+target + reference: '' + 98364940-cbc9-4676-ae98-83133f607c96: !Template answer_choices: null - id: 876e475c-7199-4ad9-8a14-dd8a367fadd9 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 98364940-cbc9-4676-ae98-83133f607c96 + jinja: 'Translate this from Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - 97c1cf42-b71c-4c97-8700-597d474defdb: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 9cf0e07b-bd5a-4c86-968c-5e36ec75cde1: !Template answer_choices: null - id: 97c1cf42-b71c-4c97-8700-597d474defdb - jinja: 'Translate into English: {{translation["ru"]}} + id: 9cf0e07b-bd5a-4c86-968c-5e36ec75cde1 + jinja: 'How do you say {{translation["en"]}} in Russian? - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - how_to_say-en-ru-target reference: '' - 9a39275f-9615-4d33-93b0-99e3d0d49c06: !Template + b29ed609-3647-4687-88c9-8356c19b0703: !Template answer_choices: null - id: 9a39275f-9615-4d33-93b0-99e3d0d49c06 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: b29ed609-3647-4687-88c9-8356c19b0703 + jinja: '{{translation["ru"]}} = English: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-source+starget - reference: '' - 9eca671d-7be7-492d-9652-4175f0c98b6d: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + bfe5a2cd-c52c-4703-929e-db381f2e2932: !Template answer_choices: null - id: 9eca671d-7be7-492d-9652-4175f0c98b6d - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: bfe5a2cd-c52c-4703-929e-db381f2e2932 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -241,131 +240,132 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - version-en-ru-target reference: '' - 9f0d7c20-f019-49af-8586-f4db2ab8b502: !Template + c78d918c-c3dc-4c53-a71c-21e54480f8a7: !Template answer_choices: null - id: 9f0d7c20-f019-49af-8586-f4db2ab8b502 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: c78d918c-c3dc-4c53-a71c-21e54480f8a7 + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target - reference: '' - a3c545e6-2570-42e1-9928-ea7044fafd11: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + d4b1889e-398d-4b01-a597-4f5e11508d40: !Template answer_choices: null - id: a3c545e6-2570-42e1-9928-ea7044fafd11 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: d4b1889e-398d-4b01-a597-4f5e11508d40 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-source+target - reference: GPT-3 paper - b44adeb1-00f6-43eb-8344-f7ecb285a315: !Template + - a_good_translation-ru-en-target + reference: '' + d5ed24af-fd55-4788-b352-114684bad41f: !Template answer_choices: null - id: b44adeb1-00f6-43eb-8344-f7ecb285a315 - jinja: '{{translation["ru"]}} = English: + id: d5ed24af-fd55-4788-b352-114684bad41f + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - bab892cc-165b-45c5-b506-b71551f9d5c1: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + d9368e60-ae18-4c89-b1fb-823cdc2bbae3: !Template answer_choices: null - id: bab892cc-165b-45c5-b506-b71551f9d5c1 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: d9368e60-ae18-4c89-b1fb-823cdc2bbae3 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - a_good_translation-ru-en-source+target reference: '' - c16ff5df-b71d-49ee-802c-2420fa01458c: !Template + ed05f70a-7dea-4eec-946f-061a2485e5a9: !Template answer_choices: null - id: c16ff5df-b71d-49ee-802c-2420fa01458c - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: ed05f70a-7dea-4eec-946f-061a2485e5a9 + jinja: 'How do you say {{translation["ru"]}} in English? - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - how_to_say-ru-en-target reference: '' - e5e1dbc2-1677-4155-b140-001f3ad9b952: !Template + f2c473df-a28e-493f-82f3-d49acf39f49b: !Template answer_choices: null - id: e5e1dbc2-1677-4155-b140-001f3ad9b952 - jinja: 'What is the English translation of : {{translation["ru"]}} + id: f2c473df-a28e-493f-82f3-d49acf39f49b + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - e6ba8d1c-4d94-481c-82ff-d3f72e34e2e9: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f71190c6-4297-4e4a-8c51-165d711c51c2: !Template answer_choices: null - id: e6ba8d1c-4d94-481c-82ff-d3f72e34e2e9 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: f71190c6-4297-4e4a-8c51-165d711c51c2 + jinja: 'Translate this into Russian: {{translation["en"]}} + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - translate-en-ru-target reference: '' - ed200eb0-0785-4c81-b4ea-16388cb562f9: !Template + f901c23f-34f6-4512-8682-e06db86e2f76: !Template answer_choices: null - id: ed200eb0-0785-4c81-b4ea-16388cb562f9 - jinja: '{{translation["ru"]}} translates into English as: + id: f901c23f-34f6-4512-8682-e06db86e2f76 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - version-en-ru-source+target reference: '' - fa56517e-e8b7-476d-a05d-e24125ee2128: !Template + f959abd8-ee89-419f-9c8e-293ec9966f68: !Template answer_choices: null - id: fa56517e-e8b7-476d-a05d-e24125ee2128 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: f959abd8-ee89-419f-9c8e-293ec9966f68 + jinja: '{{translation["ru"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - translate_as_ru-en-target reference: '' diff --git a/promptsource/templates/wmt15/cs-en/templates.yaml b/promptsource/templates/wmt15/cs-en/templates.yaml index ab8b2671e..55b07af7e 100644 --- a/promptsource/templates/wmt15/cs-en/templates.yaml +++ b/promptsource/templates/wmt15/cs-en/templates.yaml @@ -1,136 +1,136 @@ dataset: wmt15 subset: cs-en templates: - 263d7ce6-7ef1-46e0-bf0e-c63df3dab2e4: !Template + 045e175b-b7d8-4559-9807-b31c9fe976c1: !Template answer_choices: null - id: 263d7ce6-7ef1-46e0-bf0e-c63df3dab2e4 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 045e175b-b7d8-4559-9807-b31c9fe976c1 + jinja: 'English: {{translation["en"]}} = Czech: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-source+target - reference: GPT 3 paper - 2bc97934-db49-47c6-8391-7fcaa8ed502e: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 141683d7-7150-46e3-98e5-4e626b7a8eac: !Template answer_choices: null - id: 2bc97934-db49-47c6-8391-7fcaa8ed502e - jinja: ' {{translation["en"]}} translates into Czech as: + id: 141683d7-7150-46e3-98e5-4e626b7a8eac + jinja: 'What is the English translation of : {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target - reference: '' - 3d9952e5-1b15-4fba-896a-a665922628f9: !Template + - gpt-3-cs-en-target + reference: GPT-3 paper + 157bda68-3bf8-4e13-8a7a-53ad56875938: !Template answer_choices: null - id: 3d9952e5-1b15-4fba-896a-a665922628f9 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 157bda68-3bf8-4e13-8a7a-53ad56875938 + jinja: '{{translation["en"]}} = Czech: + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target - reference: '' - 5abeff8a-09ee-4908-9a87-d715e991178e: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 187ea6b9-d94c-4982-b135-9dfa865ed640: !Template answer_choices: null - id: 5abeff8a-09ee-4908-9a87-d715e991178e - jinja: 'How do you say {{translation["cs"]}} in English? + id: 187ea6b9-d94c-4982-b135-9dfa865ed640 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - version-en-cs-target reference: '' - 7dcdff65-9c00-462e-bf46-467355327d91: !Template + 422e07ed-5982-43ff-958f-8523b70a21cb: !Template answer_choices: null - id: 7dcdff65-9c00-462e-bf46-467355327d91 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 422e07ed-5982-43ff-958f-8523b70a21cb + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - translate_as_cs-en-source+target reference: '' - 809ba8b9-a275-406f-a9cb-fc2a025392c3: !Template + 549a16e8-f5d0-4364-80f1-e7406ee4ae3a: !Template answer_choices: null - id: 809ba8b9-a275-406f-a9cb-fc2a025392c3 - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: 549a16e8-f5d0-4364-80f1-e7406ee4ae3a + jinja: '{{translation["cs"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - translate_as_cs-en-target reference: '' - 867093b0-5f01-48dc-a2e5-e03bf84e92b1: !Template + 595eefb8-b237-452f-b165-f082a280faae: !Template answer_choices: null - id: 867093b0-5f01-48dc-a2e5-e03bf84e92b1 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: 595eefb8-b237-452f-b165-f082a280faae + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: GPT-3 paper - 8bb2be08-2872-4ac9-9f8f-93f54e65a98a: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 607dceeb-1034-43fc-8101-0aec68caeb7d: !Template answer_choices: null - id: 8bb2be08-2872-4ac9-9f8f-93f54e65a98a - jinja: 'English: {{translation["en"]}} = Czech: - - ||| {{translation["cs"]}}' + id: 607dceeb-1034-43fc-8101-0aec68caeb7d + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 915e8de4-588d-475a-8930-ed5e65f2109d: !Template + - a_good_translation-cs-en-source+target + reference: '' + 643bf184-cdd4-4cf5-99d6-85cbc1f20a61: !Template answer_choices: null - id: 915e8de4-588d-475a-8930-ed5e65f2109d - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: 643bf184-cdd4-4cf5-99d6-85cbc1f20a61 + jinja: 'Translate this from English into Czech: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - translate-en-cs-source+starget reference: '' - 957f1d66-dd9b-4b4e-b8ce-ddb268db329b: !Template + 66ccaa8f-7ee4-4fd3-ac02-9caf6ccd821b: !Template answer_choices: null - id: 957f1d66-dd9b-4b4e-b8ce-ddb268db329b - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 66ccaa8f-7ee4-4fd3-ac02-9caf6ccd821b + jinja: 'How do you say {{translation["en"]}} in Czech? ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -139,139 +139,140 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - how_to_say-en-cs-target reference: '' - a0649918-ebc1-437f-a90f-e7e4c5ea6e4d: !Template + 6995c2de-c2a5-43e3-9b6e-04a2a8d53ad7: !Template answer_choices: null - id: a0649918-ebc1-437f-a90f-e7e4c5ea6e4d - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 6995c2de-c2a5-43e3-9b6e-04a2a8d53ad7 + jinja: 'How do you say {{translation["cs"]}} in English? - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-source+starget + - how_to_say-cs-en-target reference: '' - a1207337-11af-4f19-a57f-cad2982cf0fd: !Template + 7287aaeb-aaab-4596-9f77-3e1bc1552be0: !Template answer_choices: null - id: a1207337-11af-4f19-a57f-cad2982cf0fd - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 7287aaeb-aaab-4596-9f77-3e1bc1552be0 + jinja: '{{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target - reference: '' - a1daba28-90ab-40f6-a222-d7adf8b98034: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 78823d0a-923e-409e-9fd4-cc43e6403956: !Template answer_choices: null - id: a1daba28-90ab-40f6-a222-d7adf8b98034 - jinja: 'Czech: {{translation["cs"]}} = English: + id: 78823d0a-923e-409e-9fd4-cc43e6403956 + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - ab23bb84-cdc6-43db-884d-e35ad3f09349: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 7aed74f4-94a2-4f64-b169-8032fc298062: !Template answer_choices: null - id: ab23bb84-cdc6-43db-884d-e35ad3f09349 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: 7aed74f4-94a2-4f64-b169-8032fc298062 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - af36aa55-d8fc-424c-8afb-bcc920d6dec2: !Template + - a_good_translation-cs-en-target + reference: '' + 7d5fe968-366f-4bc3-b744-625167a8a632: !Template answer_choices: null - id: af36aa55-d8fc-424c-8afb-bcc920d6dec2 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 7d5fe968-366f-4bc3-b744-625167a8a632 + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-source+target - reference: GPT-3 paper - af4ff1b8-3a72-4cdb-a20d-0d196c36dc53: !Template + - translate_as_en-cs-source+target + reference: '' + 876283e3-6536-4bdb-8d0d-bba32f4109bd: !Template answer_choices: null - id: af4ff1b8-3a72-4cdb-a20d-0d196c36dc53 - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 876283e3-6536-4bdb-8d0d-bba32f4109bd + jinja: 'Translate this into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - translate-en-cs-target reference: '' - d48f1328-4613-48e7-8c0a-8b0158540523: !Template + 87cecfcc-00d2-4ea8-a64d-b51a1465f187: !Template answer_choices: null - id: d48f1328-4613-48e7-8c0a-8b0158540523 - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version - should say: + id: 87cecfcc-00d2-4ea8-a64d-b51a1465f187 + jinja: 'Translate this from Czech into English: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target - reference: '' - d5809770-ad3f-444f-8c6b-1a95ebb2d0fc: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8c17680c-f022-4b7e-93d0-bdd58ab0f837: !Template answer_choices: null - id: d5809770-ad3f-444f-8c6b-1a95ebb2d0fc - jinja: 'What is the English translation of : {{translation["cs"]}} + id: 8c17680c-f022-4b7e-93d0-bdd58ab0f837 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target - reference: GPT-3 paper - e1fcc2e4-b539-494e-aa0b-943a99d31552: !Template + - version-cs-en-target + reference: '' + 8f883e31-d357-4abe-a875-dbcf372024b3: !Template answer_choices: null - id: e1fcc2e4-b539-494e-aa0b-943a99d31552 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 8f883e31-d357-4abe-a875-dbcf372024b3 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - version-cs-en-source+target reference: '' - e78c6092-63fd-453f-822c-0c975b58443f: !Template + a78ce014-6c0a-4575-943b-d7e1dd41a2f4: !Template answer_choices: null - id: e78c6092-63fd-453f-822c-0c975b58443f + id: a78ce014-6c0a-4575-943b-d7e1dd41a2f4 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Czech translation is: @@ -284,88 +285,87 @@ templates: name: !!python/tuple - a_good_translation-en-cs-source+target reference: '' - ebd16829-4538-4bd2-83c2-bbd9cc2ef967: !Template + bbb4d21d-6c03-4927-abc4-54d5a79ef525: !Template answer_choices: null - id: ebd16829-4538-4bd2-83c2-bbd9cc2ef967 - jinja: '{{translation["cs"]}} = English: + id: bbb4d21d-6c03-4927-abc4-54d5a79ef525 + jinja: 'Translate this into English: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - ebebf07e-6312-4b7b-ae7b-037723f5fba5: !Template + - translate-cs-en-target + reference: '' + d490b18b-730c-4d3f-93fc-0b3c30734b0a: !Template answer_choices: null - id: ebebf07e-6312-4b7b-ae7b-037723f5fba5 - jinja: 'Translate into Czech: {{translation["en"]}} + id: d490b18b-730c-4d3f-93fc-0b3c30734b0a + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target - reference: '' - ec94fdd4-30a6-430e-8886-94bc636c9224: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + d70ec9f3-5012-479a-ba66-d8a93f545c9c: !Template answer_choices: null - id: ec94fdd4-30a6-430e-8886-94bc636c9224 - jinja: '{{translation["cs"]}} translates into English as: - - ||| {{translation["en"]}}' + id: d70ec9f3-5012-479a-ba66-d8a93f545c9c + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - a_good_translation-en-cs-target reference: '' - ece995e4-b66d-4fab-a9db-7b63b927ae25: !Template + da5f5196-e8f6-4e11-a88d-c0647bd011b1: !Template answer_choices: null - id: ece995e4-b66d-4fab-a9db-7b63b927ae25 - jinja: '{{translation["en"]}} = Czech: + id: da5f5196-e8f6-4e11-a88d-c0647bd011b1 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f11baf07-8675-4e85-8c0d-1b4192533f75: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + dd852efc-969c-484c-a89e-a8998c5b8780: !Template answer_choices: null - id: f11baf07-8675-4e85-8c0d-1b4192533f75 - jinja: 'Translate into English: {{translation["cs"]}} + id: dd852efc-969c-484c-a89e-a8998c5b8780 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - version-en-cs-source+target reference: '' - fe81171c-916e-4250-8775-c6b7834df0ca: !Template + f25d3c08-b0d7-4a65-82d3-c6bcf498c037: !Template answer_choices: null - id: fe81171c-916e-4250-8775-c6b7834df0ca - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: f25d3c08-b0d7-4a65-82d3-c6bcf498c037 + jinja: ' {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - translate_as_en-cs-target reference: '' diff --git a/promptsource/templates/wmt15/de-en/templates.yaml b/promptsource/templates/wmt15/de-en/templates.yaml index 122300490..cfd49d8ee 100644 --- a/promptsource/templates/wmt15/de-en/templates.yaml +++ b/promptsource/templates/wmt15/de-en/templates.yaml @@ -1,166 +1,164 @@ dataset: wmt15 subset: de-en templates: - 0060361d-d7be-4181-ad60-4bf3305bd83f: !Template + 0d00f23b-5f4f-4ca0-9901-9710a23d4ed8: !Template answer_choices: null - id: 0060361d-d7be-4181-ad60-4bf3305bd83f - jinja: 'How do you say {{translation["de"]}} in English? - - ||| {{translation["en"]}}' + id: 0d00f23b-5f4f-4ca0-9901-9710a23d4ed8 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - a_good_translation-de-en-source+target reference: '' - 0f535f5b-f91d-403a-b4c8-cee3fb7bb939: !Template + 1066a885-9717-4a68-aef7-6b0f0ad1c74e: !Template answer_choices: null - id: 0f535f5b-f91d-403a-b4c8-cee3fb7bb939 - jinja: 'If the German version says: {{translation["de"]}}; then the English version + id: 1066a885-9717-4a68-aef7-6b0f0ad1c74e + jinja: 'If the original version says: {{translation["en"]}}; then the German version should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - version-en-de-target reference: '' - 111df907-6e6c-4c28-aa9f-dd4cd8874c27: !Template + 20816ff7-ba5e-4129-9a91-fc6506f0bed3: !Template answer_choices: null - id: 111df907-6e6c-4c28-aa9f-dd4cd8874c27 - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 20816ff7-ba5e-4129-9a91-fc6506f0bed3 + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - translate_as_en-de-source+target reference: '' - 14d3e540-0702-492a-be73-377b71f6f5c1: !Template + 246427e8-6adf-44d2-94a7-659d5baf4db6: !Template answer_choices: null - id: 14d3e540-0702-492a-be73-377b71f6f5c1 - jinja: ' {{translation["en"]}} translates into German as: + id: 246427e8-6adf-44d2-94a7-659d5baf4db6 + jinja: 'German: {{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - translate_as_de-en-source+target reference: '' - 26bb0ef0-b326-475a-a960-ce7feee92c53: !Template + 2528521d-91df-49c8-851f-e015b0955c91: !Template answer_choices: null - id: 26bb0ef0-b326-475a-a960-ce7feee92c53 - jinja: 'Translate English into German: {{translation["en"]}} + id: 2528521d-91df-49c8-851f-e015b0955c91 + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget - reference: '' - 3a180fcc-2e71-4f85-addc-84a7f8b15903: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 2af25b0e-1d10-4204-a82c-27d56def4ef5: !Template answer_choices: null - id: 3a180fcc-2e71-4f85-addc-84a7f8b15903 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 2af25b0e-1d10-4204-a82c-27d56def4ef5 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - version-de-en-target reference: '' - 438285d3-cb37-46b0-8f3a-2a36b69670ca: !Template + 37c9b82e-93be-4870-b416-f25c0c032e84: !Template answer_choices: null - id: 438285d3-cb37-46b0-8f3a-2a36b69670ca - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 37c9b82e-93be-4870-b416-f25c0c032e84 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-source+target - reference: GPT 3 paper - 5cd6a591-92ca-4ad4-b8b2-c4fc041b932c: !Template + - a_good_translation-en-de-source+target + reference: '' + 3bb8e23f-cdd1-4362-91ea-ab99be1e4bd8: !Template answer_choices: null - id: 5cd6a591-92ca-4ad4-b8b2-c4fc041b932c - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 3bb8e23f-cdd1-4362-91ea-ab99be1e4bd8 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target + - a_good_translation-en-de-target reference: '' - 680f17bb-ec1f-44c1-a37b-edb315f4b45a: !Template + 435c6647-3672-4d22-b30b-2a71726c5d8e: !Template answer_choices: null - id: 680f17bb-ec1f-44c1-a37b-edb315f4b45a - jinja: 'How do you say {{translation["en"]}} in German? + id: 435c6647-3672-4d22-b30b-2a71726c5d8e + jinja: 'Translate this from German into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - 72ed75ea-c7af-465c-ab4b-2cfe9571a301: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 4ea23ed8-a255-4566-8aed-9503cbc87453: !Template answer_choices: null - id: 72ed75ea-c7af-465c-ab4b-2cfe9571a301 - jinja: '{{translation["de"]}} = English: + id: 4ea23ed8-a255-4566-8aed-9503cbc87453 + jinja: 'Translate this into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 7553ecfd-104d-4bbe-b5b3-25d2baafd22f: !Template + - translate-de-en-target + reference: '' + 4f1dbc62-fa8c-4dc8-9a2e-1cad5002a38a: !Template answer_choices: null - id: 7553ecfd-104d-4bbe-b5b3-25d2baafd22f - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: 4f1dbc62-fa8c-4dc8-9a2e-1cad5002a38a + jinja: '{{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target - reference: '' - a2219a0f-c82e-44b5-bf6e-83f5eeb7a8f5: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 50a8f74f-bcf8-4a37-9cef-6c1be1f5a7b2: !Template answer_choices: null - id: a2219a0f-c82e-44b5-bf6e-83f5eeb7a8f5 + id: 50a8f74f-bcf8-4a37-9cef-6c1be1f5a7b2 jinja: 'What is the German translation of: {{translation["en"]}} ||| {{translation["de"]}}' @@ -172,10 +170,10 @@ templates: name: !!python/tuple - gpt-3-en-de-target reference: GPT-3 paper - a538680f-57b1-42ab-98f9-0a88b0b795e4: !Template + 5149a3ac-50b5-4a48-b3a4-8a28d1045d08: !Template answer_choices: null - id: a538680f-57b1-42ab-98f9-0a88b0b795e4 - jinja: 'English: {{translation["en"]}} translates into German as: + id: 5149a3ac-50b5-4a48-b3a4-8a28d1045d08 + jinja: 'English: {{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -184,126 +182,127 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target - reference: '' - b17eacfb-a235-450f-a813-d71cb8631c51: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 5490f317-04cf-413f-b3e8-17e56521bae6: !Template answer_choices: null - id: b17eacfb-a235-450f-a813-d71cb8631c51 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 5490f317-04cf-413f-b3e8-17e56521bae6 + jinja: '{{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target - reference: GPT-3 paper - b75c39a8-ba84-4934-be04-d305170da853: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5d39d71b-1297-4dac-aecc-118ea4099acb: !Template answer_choices: null - id: b75c39a8-ba84-4934-be04-d305170da853 - jinja: 'Translate into English: {{translation["de"]}} + id: 5d39d71b-1297-4dac-aecc-118ea4099acb + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - translate_as_de-en-target reference: '' - b9cdd3d3-6d03-456f-8c78-080c5b701cbb: !Template + 6441f650-a268-4249-8e47-07c2c2e599b0: !Template answer_choices: null - id: b9cdd3d3-6d03-456f-8c78-080c5b701cbb - jinja: 'German: {{translation["de"]}} translates into English as: + id: 6441f650-a268-4249-8e47-07c2c2e599b0 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - version-en-de-source+target reference: '' - c139ef45-c0f6-4cdf-a83a-d14b8c998c5c: !Template + 7963c1bc-b551-4177-adba-ebf69fc11f63: !Template answer_choices: null - id: c139ef45-c0f6-4cdf-a83a-d14b8c998c5c - jinja: 'What is the English translation of : {{translation["de"]}} + id: 7963c1bc-b551-4177-adba-ebf69fc11f63 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: GPT-3 paper - d44cfc70-e06f-4f54-b988-56de8874d577: !Template + - a_good_translation-de-en-target + reference: '' + 9730bf8b-7079-4012-9a0e-9449bcfc4145: !Template answer_choices: null - id: d44cfc70-e06f-4f54-b988-56de8874d577 - jinja: '{{translation["en"]}} = German: + id: 9730bf8b-7079-4012-9a0e-9449bcfc4145 + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d4fa63c0-9e6b-4202-a89e-94055094559d: !Template + - how_to_say-de-en-target + reference: '' + a4565b2f-b5ad-4050-8268-91222d710589: !Template answer_choices: null - id: d4fa63c0-9e6b-4202-a89e-94055094559d - jinja: 'English: {{translation["en"]}} = German: + id: a4565b2f-b5ad-4050-8268-91222d710589 + jinja: 'Translate this into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - d65aa853-5fe1-4d67-8153-d87f6c94e0e5: !Template + - translate-en-de-target + reference: '' + b306f042-211d-4c11-8614-0a78a5e2fe04: !Template answer_choices: null - id: d65aa853-5fe1-4d67-8153-d87f6c94e0e5 - jinja: 'Translate into German: {{translation["en"]}} + id: b306f042-211d-4c11-8614-0a78a5e2fe04 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - how_to_say-en-de-target reference: '' - d82f4773-0cd6-4207-8c7f-51892b525a60: !Template + dc98ad6a-8ee1-4d22-80ae-e5147d69f40e: !Template answer_choices: null - id: d82f4773-0cd6-4207-8c7f-51892b525a60 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: dc98ad6a-8ee1-4d22-80ae-e5147d69f40e + jinja: ' {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target + - translate_as_en-de-target reference: '' - da648068-e6b9-4409-b7bb-1cdae32dd866: !Template + e3801439-211f-47cc-8560-f7a5e4af96ae: !Template answer_choices: null - id: da648068-e6b9-4409-b7bb-1cdae32dd866 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: e3801439-211f-47cc-8560-f7a5e4af96ae + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -312,39 +311,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - version-de-en-source+target reference: '' - ecd049ba-a758-4e18-b0c8-49cf4422cbe4: !Template + e4f5fa03-4f06-4267-ba5c-24552360f7d5: !Template answer_choices: null - id: ecd049ba-a758-4e18-b0c8-49cf4422cbe4 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: e4f5fa03-4f06-4267-ba5c-24552360f7d5 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target - reference: '' - f6e68891-e691-4983-936b-f45574638dfe: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + e5073dfc-a334-4e03-833a-4ef16962c115: !Template answer_choices: null - id: f6e68891-e691-4983-936b-f45574638dfe - jinja: '{{translation["de"]}} translates into English as: + id: e5073dfc-a334-4e03-833a-4ef16962c115 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target - reference: '' - fbe96c98-9c98-435d-97a5-dd231fa61989: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + e65e6ca8-843d-4095-87a2-6f2609595568: !Template answer_choices: null - id: fbe96c98-9c98-435d-97a5-dd231fa61989 - jinja: 'Translate German into English: {{translation["de"]}} + id: e65e6ca8-843d-4095-87a2-6f2609595568 + jinja: 'What is the English translation of : {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -353,19 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - fd433a62-f316-4944-939a-441ec1dcc4ba: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + fac9c78f-9cae-4a4e-a0ab-8eecd2112dad: !Template answer_choices: null - id: fd433a62-f316-4944-939a-441ec1dcc4ba - jinja: 'German: {{translation["de"]}} = English: + id: fac9c78f-9cae-4a4e-a0ab-8eecd2112dad + jinja: 'Translate this from English into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 + - translate-en-de-source+starget + reference: '' diff --git a/promptsource/templates/wmt15/fi-en/templates.yaml b/promptsource/templates/wmt15/fi-en/templates.yaml index a70180b80..c660869ab 100644 --- a/promptsource/templates/wmt15/fi-en/templates.yaml +++ b/promptsource/templates/wmt15/fi-en/templates.yaml @@ -1,37 +1,37 @@ dataset: wmt15 subset: fi-en templates: - 097597be-0c94-4f70-a6e8-e01e5884a637: !Template + 072f37fd-e74d-41da-8c15-fb7bb61ce33f: !Template answer_choices: null - id: 097597be-0c94-4f70-a6e8-e01e5884a637 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 072f37fd-e74d-41da-8c15-fb7bb61ce33f + jinja: 'What is the Finnish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target - reference: '' - 099628c4-84a9-486b-bf57-a79e0cafd840: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 17b3bdcd-c560-4b96-8a8d-9aeb1e5d8103: !Template answer_choices: null - id: 099628c4-84a9-486b-bf57-a79e0cafd840 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 17b3bdcd-c560-4b96-8a8d-9aeb1e5d8103 + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target - reference: '' - 0a175591-454c-4d9e-ba4f-297c30d51980: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + 1df95437-89f7-4a1f-bb5c-aeb2ce7a182a: !Template answer_choices: null - id: 0a175591-454c-4d9e-ba4f-297c30d51980 + id: 1df95437-89f7-4a1f-bb5c-aeb2ce7a182a jinja: 'Given the following source text in English: {{translation["en"]}} , a good Finnish translation is: @@ -44,81 +44,80 @@ templates: name: !!python/tuple - a_good_translation-en-fi-source+target reference: '' - 0ce942c1-26d7-4b3c-b9ff-24ff2596a573: !Template + 1f6c7e70-f315-47b7-b92f-fc4ca443fed1: !Template answer_choices: null - id: 0ce942c1-26d7-4b3c-b9ff-24ff2596a573 - jinja: '{{translation["en"]}} = Finnish: + id: 1f6c7e70-f315-47b7-b92f-fc4ca443fed1 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 173fa2fc-3e38-483e-9674-f6d7d50c63d1: !Template + - how_to_say-fi-en-target + reference: '' + 20cd95bc-57d5-4c81-9d00-4a97cfb55ef2: !Template answer_choices: null - id: 173fa2fc-3e38-483e-9674-f6d7d50c63d1 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 20cd95bc-57d5-4c81-9d00-4a97cfb55ef2 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target - reference: '' - 3a8a540f-255c-4e79-8179-5089affbf69b: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 2db28522-02e1-4bef-9aeb-5d27ba926476: !Template answer_choices: null - id: 3a8a540f-255c-4e79-8179-5089affbf69b - jinja: 'What is the Finnish translation of: {{translation["en"]}} - - ||| {{translation["fi"]}}' + id: 2db28522-02e1-4bef-9aeb-5d27ba926476 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target - reference: GPT-3 paper - 3ddc5563-da90-4196-9292-86a709fdfd75: !Template + - a_good_translation-fi-en-source+target + reference: '' + 44dfc96b-6afb-44f6-9ede-df79e8c82295: !Template answer_choices: null - id: 3ddc5563-da90-4196-9292-86a709fdfd75 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: 44dfc96b-6afb-44f6-9ede-df79e8c82295 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target + - version-en-fi-target reference: '' - 3e5f95c4-8fe5-4dd9-960c-ff2e4b479d2b: !Template + 55f776fa-9658-476d-8c73-8fd6b0f4b9a7: !Template answer_choices: null - id: 3e5f95c4-8fe5-4dd9-960c-ff2e4b479d2b - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 55f776fa-9658-476d-8c73-8fd6b0f4b9a7 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 44d7d5db-5002-43aa-b625-5664d1d98167: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 5d63fd6d-1ab2-4da0-af39-0cfba5c3ad7b: !Template answer_choices: null - id: 44d7d5db-5002-43aa-b625-5664d1d98167 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 5d63fd6d-1ab2-4da0-af39-0cfba5c3ad7b + jinja: ' {{translation["en"]}} translates into Finnish as: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -127,69 +126,69 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - translate_as_en-fi-target reference: '' - 57e1762c-122b-4f3b-8f8e-2be3c79d0f92: !Template + 5d683d0a-a670-4bb3-90ff-8a6f059f3e2a: !Template answer_choices: null - id: 57e1762c-122b-4f3b-8f8e-2be3c79d0f92 - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 5d683d0a-a670-4bb3-90ff-8a6f059f3e2a + jinja: 'Finnish: {{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-source+starget + - translate_as_fi-en-source+target reference: '' - 5ea50f65-59de-4387-8e70-36e556d2718b: !Template + 5f065433-4860-4186-be1c-aed3d930b6fb: !Template answer_choices: null - id: 5ea50f65-59de-4387-8e70-36e556d2718b - jinja: '{{translation["fi"]}} translates into English as: + id: 5f065433-4860-4186-be1c-aed3d930b6fb + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target + - version-fi-en-target reference: '' - 625b6912-3aa8-46b7-b8f8-0d21b2fe0a23: !Template + 7519528b-39d1-45d1-911e-144615c05712: !Template answer_choices: null - id: 625b6912-3aa8-46b7-b8f8-0d21b2fe0a23 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 7519528b-39d1-45d1-911e-144615c05712 + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - translate_as_en-fi-source+target reference: '' - 6795f8d5-cec3-47f4-b1bd-55e9c0690941: !Template + 7a8d7325-d4b5-46b3-9ad4-10a3290e629e: !Template answer_choices: null - id: 6795f8d5-cec3-47f4-b1bd-55e9c0690941 - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: 7a8d7325-d4b5-46b3-9ad4-10a3290e629e + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target - reference: '' - 6b240ab5-9487-43fc-bb3e-ea5325cbcb7e: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + 9013a8e5-d4f5-4ff7-a300-fd2f1deda398: !Template answer_choices: null - id: 6b240ab5-9487-43fc-bb3e-ea5325cbcb7e - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 9013a8e5-d4f5-4ff7-a300-fd2f1deda398 + jinja: 'How do you say {{translation["en"]}} in Finnish? ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -198,174 +197,175 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - how_to_say-en-fi-target reference: '' - 75957d57-91a0-4627-be8b-02f920b1b42f: !Template + 97939122-422c-4af0-b0f7-f148fb2db9f7: !Template answer_choices: null - id: 75957d57-91a0-4627-be8b-02f920b1b42f - jinja: 'Translate into English: {{translation["fi"]}} + id: 97939122-422c-4af0-b0f7-f148fb2db9f7 + jinja: 'Translate this into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - translate-en-fi-target reference: '' - 7601409d-7187-4658-800b-99d827a1c782: !Template + 97fb6b3b-9bdc-43cb-9242-cc142e88f6a4: !Template answer_choices: null - id: 7601409d-7187-4658-800b-99d827a1c782 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - - ||| {{translation["en"]}}' + id: 97fb6b3b-9bdc-43cb-9242-cc142e88f6a4 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-source+target - reference: GPT-3 paper - 881571bd-3405-442b-8f00-7eabbba158a4: !Template + - a_good_translation-en-fi-target + reference: '' + 998043df-dc01-482d-a2b4-0fdd6bb3a961: !Template answer_choices: null - id: 881571bd-3405-442b-8f00-7eabbba158a4 - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 998043df-dc01-482d-a2b4-0fdd6bb3a961 + jinja: '{{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - translate_as_fi-en-target reference: '' - 91836597-c57e-4cf5-b007-112278f7db24: !Template + b259f666-d816-4e61-b887-faab5638ba41: !Template answer_choices: null - id: 91836597-c57e-4cf5-b007-112278f7db24 - jinja: 'How do you say {{translation["fi"]}} in English? + id: b259f666-d816-4e61-b887-faab5638ba41 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target + - version-en-fi-source+target reference: '' - 9d287a58-6ac5-42b4-8825-1645718a358d: !Template + b9707d6d-316a-4c05-b8a1-ebd0f1d5faa0: !Template answer_choices: null - id: 9d287a58-6ac5-42b4-8825-1645718a358d - jinja: 'English: {{translation["en"]}} = Finnish: + id: b9707d6d-316a-4c05-b8a1-ebd0f1d5faa0 + jinja: 'Translate this into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - afa9f710-d1da-4a94-8ffb-dcb45567b8bf: !Template + - translate-fi-en-target + reference: '' + bfce0871-2f4d-4ccb-924f-c0d4ad45603c: !Template answer_choices: null - id: afa9f710-d1da-4a94-8ffb-dcb45567b8bf - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: bfce0871-2f4d-4ccb-924f-c0d4ad45603c + jinja: 'Translate this from Finnish into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-source+target - reference: GPT 3 paper - b2d1804c-6074-4981-afd5-443e72b53743: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + c28bb50c-9efd-4a4e-aff4-7d6723d2f662: !Template answer_choices: null - id: b2d1804c-6074-4981-afd5-443e72b53743 - jinja: 'Finnish: {{translation["fi"]}} = English: + id: c28bb50c-9efd-4a4e-aff4-7d6723d2f662 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - ba4e4a2e-7c03-4ed0-8e3c-a125e7cd81cb: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + ce59f28d-c0af-40cc-bf5f-98c0ffb58bee: !Template answer_choices: null - id: ba4e4a2e-7c03-4ed0-8e3c-a125e7cd81cb - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: ce59f28d-c0af-40cc-bf5f-98c0ffb58bee + jinja: 'English: {{translation["en"]}} = Finnish: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target - reference: '' - c441750d-f758-4626-be61-d180894648c4: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + db513d6f-7cce-4231-b7b7-3b89b4f75a14: !Template answer_choices: null - id: c441750d-f758-4626-be61-d180894648c4 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: + id: db513d6f-7cce-4231-b7b7-3b89b4f75a14 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - version-fi-en-source+target reference: '' - cc6dba77-a277-4a71-b667-32636da88b88: !Template + e708a6dd-ebd3-4da5-be69-2bc8c09e0b53: !Template answer_choices: null - id: cc6dba77-a277-4a71-b667-32636da88b88 - jinja: '{{translation["fi"]}} = English: + id: e708a6dd-ebd3-4da5-be69-2bc8c09e0b53 + jinja: 'Translate this from English into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d41a01f1-66bf-4d54-9542-2e9cf1c14cf4: !Template + - translate-en-fi-source+starget + reference: '' + efacec10-b386-4cda-b887-fd1402eefc48: !Template answer_choices: null - id: d41a01f1-66bf-4d54-9542-2e9cf1c14cf4 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: efacec10-b386-4cda-b887-fd1402eefc48 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target - reference: '' - fc2489d5-89d8-41bc-ab95-4646d533d25a: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f7e9b639-b6de-4043-9cc2-a8cf15180140: !Template answer_choices: null - id: fc2489d5-89d8-41bc-ab95-4646d533d25a - jinja: 'What is the English translation of : {{translation["fi"]}} + id: f7e9b639-b6de-4043-9cc2-a8cf15180140 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: GPT-3 paper + - a_good_translation-fi-en-target + reference: '' diff --git a/promptsource/templates/wmt15/fr-en/templates.yaml b/promptsource/templates/wmt15/fr-en/templates.yaml index 9d177bcb1..01f1abace 100644 --- a/promptsource/templates/wmt15/fr-en/templates.yaml +++ b/promptsource/templates/wmt15/fr-en/templates.yaml @@ -1,53 +1,53 @@ dataset: wmt15 subset: fr-en templates: - 088f8a76-818e-461b-874f-c358123a2c7a: !Template + 048ebbe2-fc1e-492b-a9be-9ea04cd98ef0: !Template answer_choices: null - id: 088f8a76-818e-461b-874f-c358123a2c7a - jinja: 'Translate English into French: {{translation["en"]}} + id: 048ebbe2-fc1e-492b-a9be-9ea04cd98ef0 + jinja: 'What is the English translation of : {{translation["fr"]}} - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fr-source+starget - reference: '' - 0d1ef213-45e0-4fa3-aa60-56f06ab20258: !Template + - gpt-3-fr-en-target + reference: GPT-3 paper + 084baa02-db47-4d16-941a-ddcc351cd836: !Template answer_choices: null - id: 0d1ef213-45e0-4fa3-aa60-56f06ab20258 - jinja: 'If the original version says: {{translation["en"]}}; then the French version - should say: + id: 084baa02-db47-4d16-941a-ddcc351cd836 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good French translation is: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fr-target + - a_good_translation-en-fr-source+target reference: '' - 11ebba9c-de47-4415-8016-6bf84a7566eb: !Template + 099ffec8-8348-4a86-90a9-5095e19b87cf: !Template answer_choices: null - id: 11ebba9c-de47-4415-8016-6bf84a7566eb - jinja: '{{translation["fr"]}} = English: + id: 099ffec8-8348-4a86-90a9-5095e19b87cf + jinja: 'Translate this from French into English: {{translation["fr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fr-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 177bb7d6-4b09-4ba0-9717-49ca57ee9442: !Template + - translate-fr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 0e3b43b4-16af-4f03-b9c1-ad6bfd0e243b: !Template answer_choices: null - id: 177bb7d6-4b09-4ba0-9717-49ca57ee9442 - jinja: 'What is the French translation of the English sentence: {{translation["en"]}} + id: 0e3b43b4-16af-4f03-b9c1-ad6bfd0e243b + jinja: 'Translate this from English into French: {{translation["en"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -56,68 +56,68 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fr-source+target - reference: GPT 3 paper - 1e881c1f-f418-471f-908b-b9ba7a1a6ecd: !Template + - translate-en-fr-source+starget + reference: '' + 13acc84f-1bcb-442d-b96a-d087eb7d45a0: !Template answer_choices: null - id: 1e881c1f-f418-471f-908b-b9ba7a1a6ecd - jinja: 'Translate French into English: {{translation["fr"]}} + id: 13acc84f-1bcb-442d-b96a-d087eb7d45a0 + jinja: 'How do you say {{translation["en"]}} in French? - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 1ecf81db-5538-4d8b-9d16-5384b972143f: !Template + - how_to_say-en-fr-target + reference: '' + 3a6fd6dc-61a9-4c7c-acc8-57099cc3aea9: !Template answer_choices: null - id: 1ecf81db-5538-4d8b-9d16-5384b972143f - jinja: '{{translation["fr"]}} translates into English as: + id: 3a6fd6dc-61a9-4c7c-acc8-57099cc3aea9 + jinja: 'English: {{translation["en"]}} translates into French as: - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-en-target + - translate_as_en-fr-source+target reference: '' - 214598b5-16dc-4c42-8e55-e429ccd11c4d: !Template + 3c7f2aa6-63b1-4ecf-b493-a81c3e56e739: !Template answer_choices: null - id: 214598b5-16dc-4c42-8e55-e429ccd11c4d - jinja: 'Translate into French: {{translation["en"]}} + id: 3c7f2aa6-63b1-4ecf-b493-a81c3e56e739 + jinja: '{{translation["fr"]}} translates into English as: - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fr-target + - translate_as_fr-en-target reference: '' - 342fb592-d2e8-40a8-b053-f3cae6ceb2c7: !Template + 443e9acf-50df-43f4-9943-f8bdcca3bdef: !Template answer_choices: null - id: 342fb592-d2e8-40a8-b053-f3cae6ceb2c7 - jinja: 'English: {{translation["en"]}} = French: + id: 443e9acf-50df-43f4-9943-f8bdcca3bdef + jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fr-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 3891aaee-2a70-40b7-9e7c-2ad6d43e5fa1: !Template + - gpt-3-fr-en-source+target + reference: GPT-3 paper + 4ce9b78d-ed60-45cc-8125-242a2b3c5bf4: !Template answer_choices: null - id: 3891aaee-2a70-40b7-9e7c-2ad6d43e5fa1 - jinja: 'What is the English translation of the French sentence: {{translation["fr"]}} + id: 4ce9b78d-ed60-45cc-8125-242a2b3c5bf4 + jinja: 'Translate this into English: {{translation["fr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -126,54 +126,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-en-source+target - reference: GPT-3 paper - 3ccaebb0-0857-4f4c-a21c-90e4c2f19797: !Template + - translate-fr-en-target + reference: '' + 6b5bd444-941c-4a16-9094-9610ca459312: !Template answer_choices: null - id: 3ccaebb0-0857-4f4c-a21c-90e4c2f19797 - jinja: 'French: {{translation["fr"]}} = English: + id: 6b5bd444-941c-4a16-9094-9610ca459312 + jinja: 'Translate this into French: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fr-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 42003670-59f2-4113-bf67-f36d00972dca: !Template + - translate-en-fr-target + reference: '' + 6d7d61fd-cd2c-446e-a0ff-62b964d0355f: !Template answer_choices: null - id: 42003670-59f2-4113-bf67-f36d00972dca - jinja: 'If the French version says: {{translation["fr"]}}; then the English version - should say: + id: 6d7d61fd-cd2c-446e-a0ff-62b964d0355f + jinja: 'French: {{translation["fr"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fr-en-source+target + - translate_as_fr-en-source+target reference: '' - 59efc156-32fb-40d4-b639-5f02233137e0: !Template + 77c63fad-d820-4d4e-95ce-1497a6176499: !Template answer_choices: null - id: 59efc156-32fb-40d4-b639-5f02233137e0 - jinja: 'What is the English translation of : {{translation["fr"]}} + id: 77c63fad-d820-4d4e-95ce-1497a6176499 + jinja: 'If the French version says: {{translation["fr"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-en-target - reference: GPT-3 paper - 608dddb5-9544-4d1a-9a3a-6e2179ed53f5: !Template + - version-fr-en-source+target + reference: '' + 7955fc1f-6171-483f-accc-d840df1397df: !Template answer_choices: null - id: 608dddb5-9544-4d1a-9a3a-6e2179ed53f5 + id: 7955fc1f-6171-483f-accc-d840df1397df jinja: 'What is the French translation of: {{translation["en"]}} ||| {{translation["fr"]}}' @@ -185,9 +185,9 @@ templates: name: !!python/tuple - gpt-3-en-fr-target reference: GPT-3 paper - 6961a384-b948-47e4-9197-a7c8e46b5e7d: !Template + 7a84aa1f-eca4-4b85-b3f3-ea46a7c89915: !Template answer_choices: null - id: 6961a384-b948-47e4-9197-a7c8e46b5e7d + id: 7a84aa1f-eca4-4b85-b3f3-ea46a7c89915 jinja: 'If the English version says: {{translation["en"]}}; then the French version should say: @@ -200,121 +200,122 @@ templates: name: !!python/tuple - version-en-fr-source+target reference: '' - 6d5bcb5d-8f39-4543-86ae-23534bc79647: !Template + 927490ac-1935-4525-8145-17b8bfcdabe1: !Template answer_choices: null - id: 6d5bcb5d-8f39-4543-86ae-23534bc79647 - jinja: 'English: {{translation["en"]}} translates into French as: + id: 927490ac-1935-4525-8145-17b8bfcdabe1 + jinja: 'If the original version says: {{translation["en"]}}; then the French version + should say: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fr-source+target + - version-en-fr-target reference: '' - 735d779e-6cea-4453-b000-78b2c7f6c01e: !Template + c05b6121-4f46-4088-a5c5-cd1f7da75340: !Template answer_choices: null - id: 735d779e-6cea-4453-b000-78b2c7f6c01e - jinja: 'Translate into English: {{translation["fr"]}} + id: c05b6121-4f46-4088-a5c5-cd1f7da75340 + jinja: 'What is the French translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-en-target - reference: '' - 7456ba86-e183-42c5-92e7-00280adbe74c: !Template + - gpt-3-en-fr-source+target + reference: GPT 3 paper + c26db731-296e-405e-8de7-b9b49fce8407: !Template answer_choices: null - id: 7456ba86-e183-42c5-92e7-00280adbe74c - jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: c26db731-296e-405e-8de7-b9b49fce8407 + jinja: 'Given the following passage: {{translation["en"]}} , a good French translation + is: ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-en-target + - a_good_translation-en-fr-target reference: '' - 74ada6ba-9a9a-4960-aaf6-7633b9cb6f28: !Template + c5e6c439-c0b0-4405-ab3d-22de69b3a82b: !Template answer_choices: null - id: 74ada6ba-9a9a-4960-aaf6-7633b9cb6f28 - jinja: 'French: {{translation["fr"]}} translates into English as: + id: c5e6c439-c0b0-4405-ab3d-22de69b3a82b + jinja: '{{translation["en"]}} = French: - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-en-source+target - reference: '' - 8ea8f712-3c4c-49f3-ad72-f247ba0d9953: !Template + - xglm-en-fr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + cf629f0f-cba7-4623-9e98-d972b56b5b5d: !Template answer_choices: null - id: 8ea8f712-3c4c-49f3-ad72-f247ba0d9953 - jinja: '{{translation["en"]}} = French: + id: cf629f0f-cba7-4623-9e98-d972b56b5b5d + jinja: 'Given the following passage: {{translation["fr"]}} , a good English translation + is: - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fr-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9dd64acc-81cc-4594-a132-41d65caadf77: !Template + - a_good_translation-fr-en-target + reference: '' + d76db700-1b29-4ad6-aa04-17fece6a15b0: !Template answer_choices: null - id: 9dd64acc-81cc-4594-a132-41d65caadf77 - jinja: 'How do you say {{translation["en"]}} in French? + id: d76db700-1b29-4ad6-aa04-17fece6a15b0 + jinja: 'French: {{translation["fr"]}} = English: - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fr-target - reference: '' - a4e97bcf-530d-4244-a5d1-a7f0475ce671: !Template + - xglm-fr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + dba384a3-65d1-41de-96da-325a9e65a137: !Template answer_choices: null - id: a4e97bcf-530d-4244-a5d1-a7f0475ce671 - jinja: 'How do you say {{translation["fr"]}} in English? + id: dba384a3-65d1-41de-96da-325a9e65a137 + jinja: 'English: {{translation["en"]}} = French: - ||| {{translation["en"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fr-en-target - reference: '' - aa52033d-58f0-466d-b050-3cf8b5fc98c6: !Template + - xglm-en-fr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + e24ed618-f906-48d3-b9d0-9dd6c2864905: !Template answer_choices: null - id: aa52033d-58f0-466d-b050-3cf8b5fc98c6 - jinja: 'Given the following passage: {{translation["en"]}} , a good French translation - is: ||| {{translation["fr"]}}' + id: e24ed618-f906-48d3-b9d0-9dd6c2864905 + jinja: '{{translation["fr"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fr-target - reference: '' - b392064e-614f-4df4-b2ca-0637d853c72e: !Template + - xglm-fr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e971a8d9-b190-4c99-95fc-f0db73d99928: !Template answer_choices: null - id: b392064e-614f-4df4-b2ca-0637d853c72e + id: e971a8d9-b190-4c99-95fc-f0db73d99928 jinja: 'If the original version says: {{translation["fr"]}}; then the English version should say: @@ -327,9 +328,9 @@ templates: name: !!python/tuple - version-fr-en-target reference: '' - b3e92821-842c-4ae9-be87-12c805f2cbc3: !Template + f341ebe5-3f02-477b-bbc1-33a1d0750062: !Template answer_choices: null - id: b3e92821-842c-4ae9-be87-12c805f2cbc3 + id: f341ebe5-3f02-477b-bbc1-33a1d0750062 jinja: ' {{translation["en"]}} translates into French as: ||| {{translation["fr"]}}' @@ -341,24 +342,23 @@ templates: name: !!python/tuple - translate_as_en-fr-target reference: '' - cc0e7dfc-5308-4560-bcfe-eb95511e1868: !Template + f7e45e6b-e002-4cdf-a0f2-40b38f678697: !Template answer_choices: null - id: cc0e7dfc-5308-4560-bcfe-eb95511e1868 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good French translation is: + id: f7e45e6b-e002-4cdf-a0f2-40b38f678697 + jinja: 'How do you say {{translation["fr"]}} in English? - ||| {{translation["fr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fr-source+target + - how_to_say-fr-en-target reference: '' - fdebd9e6-ac4a-4679-b725-26c3286c5ef1: !Template + f960ea4e-fd15-4c7d-baf5-139ccd5ca6b3: !Template answer_choices: null - id: fdebd9e6-ac4a-4679-b725-26c3286c5ef1 + id: f960ea4e-fd15-4c7d-baf5-139ccd5ca6b3 jinja: 'Given the following source text in French: {{translation["fr"]}} , a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata diff --git a/promptsource/templates/wmt15/ru-en/templates.yaml b/promptsource/templates/wmt15/ru-en/templates.yaml index bc4a975bf..6dbedaf75 100644 --- a/promptsource/templates/wmt15/ru-en/templates.yaml +++ b/promptsource/templates/wmt15/ru-en/templates.yaml @@ -1,37 +1,37 @@ dataset: wmt15 subset: ru-en templates: - 03df7ac2-16f5-44c8-87c2-b5223a4c94e9: !Template + 1f0d9121-0b7e-4e3c-9a85-b7a0ccdb19d6: !Template answer_choices: null - id: 03df7ac2-16f5-44c8-87c2-b5223a4c94e9 - jinja: 'Russian: {{translation["ru"]}} = English: - - ||| {{translation["en"]}}' + id: 1f0d9121-0b7e-4e3c-9a85-b7a0ccdb19d6 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 089ff29c-4e66-45f8-bf87-95aa1768482b: !Template + - a_good_translation-ru-en-source+target + reference: '' + 232268c9-bd22-4928-9ebf-db6a91cb63a8: !Template answer_choices: null - id: 089ff29c-4e66-45f8-bf87-95aa1768482b - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 232268c9-bd22-4928-9ebf-db6a91cb63a8 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - 0dd5c911-ec29-4000-a486-ecf5f4057fc4: !Template + - a_good_translation-ru-en-target + reference: '' + 25ab3b07-66e0-4a1a-80e1-dc710f140595: !Template answer_choices: null - id: 0dd5c911-ec29-4000-a486-ecf5f4057fc4 + id: 25ab3b07-66e0-4a1a-80e1-dc710f140595 jinja: 'English: {{translation["en"]}} = Russian: ||| {{translation["ru"]}}' @@ -43,24 +43,26 @@ templates: name: !!python/tuple - xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 193b2702-e0c2-4189-b7b4-3ab2417f2a07: !Template + 33fb869b-2517-4761-a1d9-66de39c8362a: !Template answer_choices: null - id: 193b2702-e0c2-4189-b7b4-3ab2417f2a07 - jinja: 'How do you say {{translation["ru"]}} in English? + id: 33fb869b-2517-4761-a1d9-66de39c8362a + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - version-en-ru-target reference: '' - 19850a56-1fe3-4e43-a78d-3a001a0ca1eb: !Template + 4557e646-2161-46c9-a4a6-ad93ffcf061b: !Template answer_choices: null - id: 19850a56-1fe3-4e43-a78d-3a001a0ca1eb - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 4557e646-2161-46c9-a4a6-ad93ffcf061b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -69,71 +71,68 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - a_good_translation-en-ru-source+target reference: '' - 1f9b14d6-974c-4f13-9ee3-fc6ea0978ff4: !Template + 46d28534-cdcf-4945-a1d8-2369f20d6eed: !Template answer_choices: null - id: 1f9b14d6-974c-4f13-9ee3-fc6ea0978ff4 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 46d28534-cdcf-4945-a1d8-2369f20d6eed + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target - reference: '' - 24a43c12-fbc5-48f1-afe1-099a978a0c6a: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 4d9c4540-cfd7-4b06-91ef-cb4391e823e6: !Template answer_choices: null - id: 24a43c12-fbc5-48f1-afe1-099a978a0c6a - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 4d9c4540-cfd7-4b06-91ef-cb4391e823e6 + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - translate_as_ru-en-target reference: '' - 26881ddf-0d5d-42ef-ad3c-52bfd8cf861d: !Template + 60a5e016-58ad-492c-8b23-0a1d9e3a1652: !Template answer_choices: null - id: 26881ddf-0d5d-42ef-ad3c-52bfd8cf861d - jinja: '{{translation["en"]}} = Russian: + id: 60a5e016-58ad-492c-8b23-0a1d9e3a1652 + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 271ddd3e-2f20-41b9-8ca3-de0ba9337214: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 71527c51-6dcd-4a0e-b31a-1e013b381911: !Template answer_choices: null - id: 271ddd3e-2f20-41b9-8ca3-de0ba9337214 - jinja: '{{translation["ru"]}} = English: - - ||| {{translation["en"]}}' + id: 71527c51-6dcd-4a0e-b31a-1e013b381911 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 2e262e52-3377-4764-bba5-f2f7d2a90bbd: !Template + - a_good_translation-en-ru-target + reference: '' + 743a939d-388f-4af6-b67e-d7a527beb9f5: !Template answer_choices: null - id: 2e262e52-3377-4764-bba5-f2f7d2a90bbd - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 743a939d-388f-4af6-b67e-d7a527beb9f5 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -142,192 +141,195 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - version-en-ru-source+target reference: '' - 3cbb11b5-a323-411d-b5c9-2ecf951f67b2: !Template + 828c119b-bee4-4d63-bb2b-6641bcc65d4b: !Template answer_choices: null - id: 3cbb11b5-a323-411d-b5c9-2ecf951f67b2 - jinja: 'Translate into Russian: {{translation["en"]}} + id: 828c119b-bee4-4d63-bb2b-6641bcc65d4b + jinja: 'Translate this into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - translate-ru-en-target reference: '' - 54c91c79-b709-40aa-85d6-68999f351fb3: !Template + 86784af8-e562-4f24-b31a-58498e7f504c: !Template answer_choices: null - id: 54c91c79-b709-40aa-85d6-68999f351fb3 - jinja: 'Translate into English: {{translation["ru"]}} + id: 86784af8-e562-4f24-b31a-58498e7f504c + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - version-ru-en-source+target reference: '' - 69a4c8a9-8d0c-4900-a176-c2c03893cba0: !Template + 88e0b171-ad44-4016-86dd-c255a9a41bbc: !Template answer_choices: null - id: 69a4c8a9-8d0c-4900-a176-c2c03893cba0 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 88e0b171-ad44-4016-86dd-c255a9a41bbc + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 7443b4b0-34e6-4faf-8e3b-119383aba078: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 8df21afa-d98a-46a3-8976-a8609c94685b: !Template answer_choices: null - id: 7443b4b0-34e6-4faf-8e3b-119383aba078 - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: 8df21afa-d98a-46a3-8976-a8609c94685b + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target - reference: '' - 7828e715-1e1c-40b3-aa38-88b2f1281b18: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 975e05e6-4a83-45b3-b9ab-91eaad7e7be9: !Template answer_choices: null - id: 7828e715-1e1c-40b3-aa38-88b2f1281b18 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 975e05e6-4a83-45b3-b9ab-91eaad7e7be9 + jinja: 'What is the Russian translation of: {{translation["en"]}} + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target - reference: '' - 89c69c2e-df37-46b2-8679-09e3ab676dfd: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 993c7951-af88-4162-a007-070e297bffac: !Template answer_choices: null - id: 89c69c2e-df37-46b2-8679-09e3ab676dfd - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 993c7951-af88-4162-a007-070e297bffac + jinja: 'Translate this from English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - translate-en-ru-source+starget reference: '' - abe216ca-9422-4a33-ac6f-10901b696cfa: !Template + 9f689169-2eee-4910-b8da-0913caceb1db: !Template answer_choices: null - id: abe216ca-9422-4a33-ac6f-10901b696cfa - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 9f689169-2eee-4910-b8da-0913caceb1db + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-source+starget - reference: '' - ba114e8a-d16c-406d-a67f-e0add4293708: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + aaf1e960-8b2f-4550-95c9-9ac618c26b1a: !Template answer_choices: null - id: ba114e8a-d16c-406d-a67f-e0add4293708 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: aaf1e960-8b2f-4550-95c9-9ac618c26b1a + jinja: 'How do you say {{translation["ru"]}} in English? - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - c0e0b76b-ad5c-4d9b-a76d-8845c10f72ae: !Template + - how_to_say-ru-en-target + reference: '' + b87d7baa-985e-439c-8d19-6c22cab8e0dd: !Template answer_choices: null - id: c0e0b76b-ad5c-4d9b-a76d-8845c10f72ae - jinja: ' {{translation["en"]}} translates into Russian as: + id: b87d7baa-985e-439c-8d19-6c22cab8e0dd + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate_as_ru-en-source+target reference: '' - c414fb01-ded7-46a3-a86c-d313f04e449b: !Template + c00e71ea-ce5e-4559-923e-5a8d91ad3bc9: !Template answer_choices: null - id: c414fb01-ded7-46a3-a86c-d313f04e449b - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: c00e71ea-ce5e-4559-923e-5a8d91ad3bc9 + jinja: ' {{translation["en"]}} translates into Russian as: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - translate_as_en-ru-target reference: '' - c84f5647-6d22-4e23-b7a0-13fbc5a3345f: !Template + ce31c299-7ff7-44ac-a120-d0c0db44ff10: !Template answer_choices: null - id: c84f5647-6d22-4e23-b7a0-13fbc5a3345f - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: ce31c299-7ff7-44ac-a120-d0c0db44ff10 + jinja: 'How do you say {{translation["en"]}} in Russian? - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: GPT-3 paper - cf4680f0-7c08-4fcb-b6ea-ac03b143894c: !Template + - how_to_say-en-ru-target + reference: '' + d2d02ff0-dcfe-4fdd-a774-e00e75121629: !Template answer_choices: null - id: cf4680f0-7c08-4fcb-b6ea-ac03b143894c - jinja: 'How do you say {{translation["en"]}} in Russian? + id: d2d02ff0-dcfe-4fdd-a774-e00e75121629 + jinja: 'Translate this from Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target - reference: '' - db532de1-be7e-479b-965b-ab2f6a34f01c: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + d6099b4c-f38a-49e1-9666-c6cd51d27115: !Template answer_choices: null - id: db532de1-be7e-479b-965b-ab2f6a34f01c - jinja: '{{translation["ru"]}} translates into English as: + id: d6099b4c-f38a-49e1-9666-c6cd51d27115 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - version-ru-en-target reference: '' - ee253aa4-b21e-47fc-87e1-d033b9d4a5f9: !Template + dafb6762-44ba-4635-9c08-211efd746eec: !Template answer_choices: null - id: ee253aa4-b21e-47fc-87e1-d033b9d4a5f9 + id: dafb6762-44ba-4635-9c08-211efd746eec jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} ||| {{translation["en"]}}' @@ -339,11 +341,10 @@ templates: name: !!python/tuple - gpt-3-ru-en-source+target reference: GPT-3 paper - f21d7b09-21cd-426d-b323-cccb6a29db04: !Template + e8a591e9-af99-4144-975a-1c05d692a6a6: !Template answer_choices: null - id: f21d7b09-21cd-426d-b323-cccb6a29db04 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: e8a591e9-af99-4144-975a-1c05d692a6a6 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -352,20 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - translate_as_en-ru-source+target reference: '' - fd9c5bcf-5c69-4af6-905e-ae08247b4278: !Template + f5796829-56c0-45cb-8036-a536b4f38409: !Template answer_choices: null - id: fd9c5bcf-5c69-4af6-905e-ae08247b4278 - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: f5796829-56c0-45cb-8036-a536b4f38409 + jinja: 'Translate this into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target + - translate-en-ru-target reference: '' diff --git a/promptsource/templates/wmt16/cs-en/templates.yaml b/promptsource/templates/wmt16/cs-en/templates.yaml index bf5d9c0be..b25b2c900 100644 --- a/promptsource/templates/wmt16/cs-en/templates.yaml +++ b/promptsource/templates/wmt16/cs-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt16 subset: cs-en templates: - 0a23584b-3021-44e5-a3db-6f47ce2d521e: !Template + 01f10092-5894-48ef-8538-f7aa14656c65: !Template answer_choices: null - id: 0a23584b-3021-44e5-a3db-6f47ce2d521e - jinja: '{{translation["cs"]}} translates into English as: + id: 01f10092-5894-48ef-8538-f7aa14656c65 + jinja: '{{translation["cs"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -13,95 +13,98 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target - reference: '' - 0b2c143d-d2e5-4657-9848-e7ed35600074: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 06f4d626-b17b-431f-9a28-b06307824ebe: !Template answer_choices: null - id: 0b2c143d-d2e5-4657-9848-e7ed35600074 - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 06f4d626-b17b-431f-9a28-b06307824ebe + jinja: 'Translate this into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-source+target - reference: GPT-3 paper - 176357b9-af45-4380-89fb-e53b82198c76: !Template + - translate-en-cs-target + reference: '' + 09d83b14-f487-4ddc-8096-5026e7e5dcd9: !Template answer_choices: null - id: 176357b9-af45-4380-89fb-e53b82198c76 - jinja: '{{translation["en"]}} = Czech: + id: 09d83b14-f487-4ddc-8096-5026e7e5dcd9 + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 188e0622-6a3a-49ff-89c9-7c3b93cbabfc: !Template + - version-en-cs-target + reference: '' + 12eaeafb-14f3-4852-8752-949a9e5dd782: !Template answer_choices: null - id: 188e0622-6a3a-49ff-89c9-7c3b93cbabfc - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 12eaeafb-14f3-4852-8752-949a9e5dd782 + jinja: 'What is the Czech translation of: {{translation["en"]}} + + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target - reference: '' - 18d8b05a-25db-45ab-b383-5ad387e6c3cb: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 22053344-0169-4a3b-a021-bde387866fa6: !Template answer_choices: null - id: 18d8b05a-25db-45ab-b383-5ad387e6c3cb - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: 22053344-0169-4a3b-a021-bde387866fa6 + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - translate_as_en-cs-source+target reference: '' - 1b264c9e-1326-4754-ba69-ce63173523ca: !Template + 3037d57c-496d-4b2c-9f96-4965092d0a9d: !Template answer_choices: null - id: 1b264c9e-1326-4754-ba69-ce63173523ca - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 3037d57c-496d-4b2c-9f96-4965092d0a9d + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-source+target - reference: GPT 3 paper - 4d9cd1b4-7af8-46f8-a9ad-d5d888e97c19: !Template + - version-en-cs-source+target + reference: '' + 4b8ac0e0-c902-4814-9dae-23f09db04f1d: !Template answer_choices: null - id: 4d9cd1b4-7af8-46f8-a9ad-d5d888e97c19 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 4b8ac0e0-c902-4814-9dae-23f09db04f1d + jinja: 'Czech: {{translation["cs"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - translate_as_cs-en-source+target reference: '' - 6b0c4291-aa98-4960-9954-e43d003ac005: !Template + 5600ba5b-2d7d-47ef-9f51-381dc6b8a2c6: !Template answer_choices: null - id: 6b0c4291-aa98-4960-9954-e43d003ac005 - jinja: 'What is the Czech translation of: {{translation["en"]}} + id: 5600ba5b-2d7d-47ef-9f51-381dc6b8a2c6 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -110,27 +113,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: GPT-3 paper - 7bdf903f-6dd7-49fa-bffd-f493130dacc5: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + 5a68c1d5-1780-4ae9-9d2b-de7bd20b359b: !Template answer_choices: null - id: 7bdf903f-6dd7-49fa-bffd-f493130dacc5 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: 5a68c1d5-1780-4ae9-9d2b-de7bd20b359b + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - version-cs-en-source+target reference: '' - 7cc9b4f1-429c-4435-b231-4a92f27dc839: !Template + 6fba80b6-98f0-470d-b8d3-0dcfd0a51c76: !Template answer_choices: null - id: 7cc9b4f1-429c-4435-b231-4a92f27dc839 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Czech translation is: + id: 6fba80b6-98f0-470d-b8d3-0dcfd0a51c76 + jinja: 'English: {{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -139,41 +142,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-source+target - reference: '' - 7de01c31-1280-4ac5-b880-77bc539538fe: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 74ec8367-77bf-4514-8a5b-831797eff523: !Template answer_choices: null - id: 7de01c31-1280-4ac5-b880-77bc539538fe - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 74ec8367-77bf-4514-8a5b-831797eff523 + jinja: '{{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_cs-en-target reference: '' - 83b65581-14ea-4d11-b898-5289ab641dfa: !Template + 7de95a1a-a966-4126-b570-48827f077a4b: !Template answer_choices: null - id: 83b65581-14ea-4d11-b898-5289ab641dfa - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 7de95a1a-a966-4126-b570-48827f077a4b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - a_good_translation-en-cs-source+target reference: '' - 9d85919c-6750-4c8f-b1d6-c4575c1f9333: !Template + 8c8b8182-ed7f-489d-a58f-9617dada0e8c: !Template answer_choices: null - id: 9d85919c-6750-4c8f-b1d6-c4575c1f9333 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 8c8b8182-ed7f-489d-a58f-9617dada0e8c + jinja: 'Translate this from English into Czech: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -184,125 +187,123 @@ templates: name: !!python/tuple - translate-en-cs-source+starget reference: '' - a565febb-6876-4f4c-96cc-2239ca19cd9d: !Template + 8dbff6c3-6ac0-42b9-a749-4283bd209449: !Template answer_choices: null - id: a565febb-6876-4f4c-96cc-2239ca19cd9d - jinja: 'Translate Czech into English: {{translation["cs"]}} - - ||| {{translation["en"]}}' + id: 8dbff6c3-6ac0-42b9-a749-4283bd209449 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - a59f9aa4-48c4-420d-98d2-91739afcb2fe: !Template + - a_good_translation-en-cs-target + reference: '' + 94a10abc-f80f-4368-b1ec-2569eadf8b59: !Template answer_choices: null - id: a59f9aa4-48c4-420d-98d2-91739afcb2fe - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: 94a10abc-f80f-4368-b1ec-2569eadf8b59 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target - reference: '' - accb80ac-3784-450b-88e7-afd3c8b161d7: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + a1ee72ef-4af0-4492-abbd-c7792e59aa77: !Template answer_choices: null - id: accb80ac-3784-450b-88e7-afd3c8b161d7 - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: a1ee72ef-4af0-4492-abbd-c7792e59aa77 + jinja: ' {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - translate_as_en-cs-target reference: '' - b19dedc6-20ef-47d3-b755-fb7bf0e11001: !Template + b483a635-8ee9-47dc-9607-84dd22ba575e: !Template answer_choices: null - id: b19dedc6-20ef-47d3-b755-fb7bf0e11001 - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version - should say: + id: b483a635-8ee9-47dc-9607-84dd22ba575e + jinja: 'Translate this into English: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target + - translate-cs-en-target reference: '' - b4c94927-68ae-4ef4-bbd1-1c2c986cb504: !Template + b653b9fc-1f3a-4eb4-a93f-42d5bd624abe: !Template answer_choices: null - id: b4c94927-68ae-4ef4-bbd1-1c2c986cb504 - jinja: 'Translate into English: {{translation["cs"]}} + id: b653b9fc-1f3a-4eb4-a93f-42d5bd624abe + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target - reference: '' - b9232185-faa3-4c04-857b-81d1b9866d3f: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c8197dd1-a836-472b-a703-6ed741cc735f: !Template answer_choices: null - id: b9232185-faa3-4c04-857b-81d1b9866d3f - jinja: 'English: {{translation["en"]}} = Czech: + id: c8197dd1-a836-472b-a703-6ed741cc735f + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - ba9ef45a-e303-4b08-87fb-c7dd81ebdf98: !Template + - a_good_translation-cs-en-target + reference: '' + c83d3b75-f46e-4f24-8305-7f495f6c2546: !Template answer_choices: null - id: ba9ef45a-e303-4b08-87fb-c7dd81ebdf98 - jinja: ' {{translation["en"]}} translates into Czech as: + id: c83d3b75-f46e-4f24-8305-7f495f6c2546 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - version-cs-en-target reference: '' - bc27e6a2-b096-43a5-9443-5a285cd0f7b0: !Template + ce513163-bf94-419a-bf2f-fdef00124e8f: !Template answer_choices: null - id: bc27e6a2-b096-43a5-9443-5a285cd0f7b0 - jinja: 'Translate into Czech: {{translation["en"]}} + id: ce513163-bf94-419a-bf2f-fdef00124e8f + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target + - how_to_say-en-cs-target reference: '' - d4c295d1-7f94-4e8d-a841-437f1fb3a8e5: !Template + ddd2c29e-ca72-4f16-9189-c014a19126e1: !Template answer_choices: null - id: d4c295d1-7f94-4e8d-a841-437f1fb3a8e5 - jinja: '{{translation["cs"]}} = English: + id: ddd2c29e-ca72-4f16-9189-c014a19126e1 + jinja: 'How do you say {{translation["cs"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,39 +312,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - da65d991-c483-4ce2-aa07-93b2588c4512: !Template + - how_to_say-cs-en-target + reference: '' + e3b1359f-d3f8-4dc1-977b-bb1d72066a4d: !Template answer_choices: null - id: da65d991-c483-4ce2-aa07-93b2588c4512 - jinja: 'How do you say {{translation["cs"]}} in English? + id: e3b1359f-d3f8-4dc1-977b-bb1d72066a4d + jinja: 'Translate this from Czech into English: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target - reference: '' - f4aa9206-271a-4e3b-94f7-b305b53eaabd: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + ee26f75c-9ac8-4f60-a8a4-9b517c39e27e: !Template answer_choices: null - id: f4aa9206-271a-4e3b-94f7-b305b53eaabd - jinja: 'English: {{translation["en"]}} translates into Czech as: - - ||| {{translation["cs"]}}' + id: ee26f75c-9ac8-4f60-a8a4-9b517c39e27e + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - a_good_translation-cs-en-source+target reference: '' - f6143402-e4d4-494b-826d-b2733a9f1690: !Template + f05efb50-36c1-4e9c-a074-1feaa78c9868: !Template answer_choices: null - id: f6143402-e4d4-494b-826d-b2733a9f1690 + id: f05efb50-36c1-4e9c-a074-1feaa78c9868 jinja: 'What is the English translation of : {{translation["cs"]}} ||| {{translation["en"]}}' @@ -355,17 +355,17 @@ templates: name: !!python/tuple - gpt-3-cs-en-target reference: GPT-3 paper - f6dcc950-8166-4806-b277-1d251c11a2fb: !Template + f25d54ca-cee7-4e06-b67f-6be82672707d: !Template answer_choices: null - id: f6dcc950-8166-4806-b277-1d251c11a2fb - jinja: 'Czech: {{translation["cs"]}} = English: + id: f25d54ca-cee7-4e06-b67f-6be82672707d + jinja: '{{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-source+target + - xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/de-en/templates.yaml b/promptsource/templates/wmt16/de-en/templates.yaml index ff840bc88..62e81ec72 100644 --- a/promptsource/templates/wmt16/de-en/templates.yaml +++ b/promptsource/templates/wmt16/de-en/templates.yaml @@ -1,54 +1,52 @@ dataset: wmt16 subset: de-en templates: - 011eb653-452c-4e29-bc28-41ead1f410b3: !Template + 29cd97a5-6b9d-401d-93e7-73300051984e: !Template answer_choices: null - id: 011eb653-452c-4e29-bc28-41ead1f410b3 - jinja: 'Translate into German: {{translation["en"]}} + id: 29cd97a5-6b9d-401d-93e7-73300051984e + jinja: 'Translate this into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - translate-de-en-target reference: '' - 1193db70-93fd-4c9e-aa73-74d2b89d376b: !Template + 2f8a0523-7036-4696-bb89-bf5d69f5993e: !Template answer_choices: null - id: 1193db70-93fd-4c9e-aa73-74d2b89d376b - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 2f8a0523-7036-4696-bb89-bf5d69f5993e + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target - reference: GPT-3 paper - 131d4804-382c-4610-aae0-69ddd20bfa31: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 30177c30-223b-4919-850e-744a32606d6a: !Template answer_choices: null - id: 131d4804-382c-4610-aae0-69ddd20bfa31 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: 30177c30-223b-4919-850e-744a32606d6a + jinja: 'German: {{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - 16c3235a-aafd-4800-bc4e-010e9244af53: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 388eff35-8559-406c-a733-14d2ad09dd52: !Template answer_choices: null - id: 16c3235a-aafd-4800-bc4e-010e9244af53 - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 388eff35-8559-406c-a733-14d2ad09dd52 + jinja: '{{translation["de"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -57,54 +55,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - translate_as_de-en-target reference: '' - 1d30ca18-503b-4c06-8f1a-ec53213d4d96: !Template + 38a01751-b14a-42bb-896f-29ad7ea993f3: !Template answer_choices: null - id: 1d30ca18-503b-4c06-8f1a-ec53213d4d96 - jinja: 'How do you say {{translation["de"]}} in English? - - ||| {{translation["en"]}}' + id: 38a01751-b14a-42bb-896f-29ad7ea993f3 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - a_good_translation-de-en-source+target reference: '' - 1edbc64c-ee40-4c53-aa7a-78593b2b9366: !Template + 4063ab3f-5467-4842-b445-a65e67c6c7e0: !Template answer_choices: null - id: 1edbc64c-ee40-4c53-aa7a-78593b2b9366 - jinja: 'German: {{translation["de"]}} = English: + id: 4063ab3f-5467-4842-b445-a65e67c6c7e0 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 333f3945-9187-4e8c-9eb9-c26b6691089e: !Template + - version-en-de-target + reference: '' + 42e7588b-a31d-4e83-a09d-642a943ca4e2: !Template answer_choices: null - id: 333f3945-9187-4e8c-9eb9-c26b6691089e - jinja: 'How do you say {{translation["en"]}} in German? + id: 42e7588b-a31d-4e83-a09d-642a943ca4e2 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - 5b00be40-5344-43d6-b988-db3fefca8adf: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 5476dadf-95bf-4978-82c4-bf30708f00b5: !Template answer_choices: null - id: 5b00be40-5344-43d6-b988-db3fefca8adf - jinja: 'English: {{translation["en"]}} translates into German as: + id: 5476dadf-95bf-4978-82c4-bf30708f00b5 + jinja: ' {{translation["en"]}} translates into German as: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -113,81 +111,81 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - translate_as_en-de-target reference: '' - 5dad0b28-42ae-4adf-8ca6-9065da30840b: !Template + 66bab156-55dd-4675-83ec-a70d5bf9dd9d: !Template answer_choices: null - id: 5dad0b28-42ae-4adf-8ca6-9065da30840b - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 66bab156-55dd-4675-83ec-a70d5bf9dd9d + jinja: 'How do you say {{translation["de"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target + - how_to_say-de-en-target reference: '' - 6476968d-e278-42ff-9f0a-48d705c58b88: !Template + 6c3bd7a7-3c2c-47e9-b9da-f760d3b9ee02: !Template answer_choices: null - id: 6476968d-e278-42ff-9f0a-48d705c58b88 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 6c3bd7a7-3c2c-47e9-b9da-f760d3b9ee02 + jinja: 'Translate this into German: {{translation["en"]}} + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate-en-de-target reference: '' - 7ae0f06d-66a2-4e43-b993-0e2933b0acfc: !Template + 6f6c002f-4f20-42a1-ad4c-2d34aa892c75: !Template answer_choices: null - id: 7ae0f06d-66a2-4e43-b993-0e2933b0acfc - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: 6f6c002f-4f20-42a1-ad4c-2d34aa892c75 + jinja: 'What is the German translation of: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target - reference: '' - 8aa85abc-974d-47ee-9b99-e534b100a77e: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + 87f1f54f-ff9e-4eae-94db-9d260fe1965a: !Template answer_choices: null - id: 8aa85abc-974d-47ee-9b99-e534b100a77e - jinja: '{{translation["de"]}} translates into English as: + id: 87f1f54f-ff9e-4eae-94db-9d260fe1965a + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - translate_as_en-de-source+target reference: '' - 8aec39a8-d6c1-42c9-b27e-c03c1bea40f7: !Template + 89f344f9-869d-41c9-a647-1adadbb9e7e1: !Template answer_choices: null - id: 8aec39a8-d6c1-42c9-b27e-c03c1bea40f7 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 89f344f9-869d-41c9-a647-1adadbb9e7e1 + jinja: 'Translate this from English into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - translate-en-de-source+starget reference: '' - 955fd7a7-7e76-4983-9f89-864b31955076: !Template + 95e7c43a-e477-410c-a50f-b64cef3ed1e9: !Template answer_choices: null - id: 955fd7a7-7e76-4983-9f89-864b31955076 + id: 95e7c43a-e477-410c-a50f-b64cef3ed1e9 jinja: 'What is the German translation of the English sentence: {{translation["en"]}} ||| {{translation["de"]}}' @@ -199,24 +197,10 @@ templates: name: !!python/tuple - gpt-3-en-de-source+target reference: GPT 3 paper - 95a4ed85-ee64-42fe-b9fa-15d204d1f240: !Template + 995c6ce7-020e-40d1-a30f-9a526e1f841c: !Template answer_choices: null - id: 95a4ed85-ee64-42fe-b9fa-15d204d1f240 - jinja: 'What is the German translation of: {{translation["en"]}} - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - gpt-3-en-de-target - reference: GPT-3 paper - 9d5035f4-84f5-437e-9753-c5a3f2f6ce85: !Template - answer_choices: null - id: 9d5035f4-84f5-437e-9753-c5a3f2f6ce85 - jinja: ' {{translation["en"]}} translates into German as: + id: 995c6ce7-020e-40d1-a30f-9a526e1f841c + jinja: 'How do you say {{translation["en"]}} in German? ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -225,54 +209,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - how_to_say-en-de-target reference: '' - b2a5be40-6fee-4fd8-b585-991e79464f5c: !Template + a3d29f52-0a5c-4250-8fb4-b36009602c9e: !Template answer_choices: null - id: b2a5be40-6fee-4fd8-b585-991e79464f5c - jinja: 'Translate into English: {{translation["de"]}} + id: a3d29f52-0a5c-4250-8fb4-b36009602c9e + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - a_good_translation-de-en-target reference: '' - cbb1cd17-b662-407d-b636-bf37995656e1: !Template + a3e46f64-557e-4fb0-94d6-e0961b7b2fb7: !Template answer_choices: null - id: cbb1cd17-b662-407d-b636-bf37995656e1 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: a3e46f64-557e-4fb0-94d6-e0961b7b2fb7 + jinja: 'Translate this from German into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target - reference: '' - cec6977f-bd0a-4e11-8ca4-ab8858a69502: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + b07c1795-16c5-4389-b6e9-8292a8bc54d6: !Template answer_choices: null - id: cec6977f-bd0a-4e11-8ca4-ab8858a69502 - jinja: 'German: {{translation["de"]}} translates into English as: + id: b07c1795-16c5-4389-b6e9-8292a8bc54d6 + jinja: '{{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target - reference: '' - d28e05c8-3bff-4579-a173-ae39b1e4eec1: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c47dfcfd-1d10-4744-875a-2a11eb28fa17: !Template answer_choices: null - id: d28e05c8-3bff-4579-a173-ae39b1e4eec1 + id: c47dfcfd-1d10-4744-875a-2a11eb28fa17 jinja: '{{translation["de"]}} = English: ||| {{translation["en"]}}' @@ -284,24 +268,25 @@ templates: name: !!python/tuple - xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - d8546622-b890-467e-9a2a-1fbefb9239a6: !Template + c5be4a28-f5cf-4444-80c0-0c67bcc283d6: !Template answer_choices: null - id: d8546622-b890-467e-9a2a-1fbefb9239a6 - jinja: 'Translate English into German: {{translation["en"]}} + id: c5be4a28-f5cf-4444-80c0-0c67bcc283d6 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget + - version-de-en-target reference: '' - e13131f0-bc41-4e29-a552-e2f3e99abffe: !Template + c9ec611f-ec0e-4c97-ad56-71f3aeef5e41: !Template answer_choices: null - id: e13131f0-bc41-4e29-a552-e2f3e99abffe - jinja: 'Translate German into English: {{translation["de"]}} + id: c9ec611f-ec0e-4c97-ad56-71f3aeef5e41 + jinja: 'What is the English translation of : {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -310,25 +295,24 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e397bc2f-425e-4afe-a543-c8973054ef52: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + d45b599a-2f9d-4d56-a7cf-da02ac2a16e4: !Template answer_choices: null - id: e397bc2f-425e-4afe-a543-c8973054ef52 - jinja: 'English: {{translation["en"]}} = German: - - ||| {{translation["de"]}}' + id: d45b599a-2f9d-4d56-a7cf-da02ac2a16e4 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - e7d42462-80ec-498e-916a-56ffb65dfb8c: !Template + - a_good_translation-en-de-target + reference: '' + e7232521-1517-44ec-8a51-85f5db80c1eb: !Template answer_choices: null - id: e7d42462-80ec-498e-916a-56ffb65dfb8c + id: e7232521-1517-44ec-8a51-85f5db80c1eb jinja: 'Given the following source text in English: {{translation["en"]}} , a good German translation is: @@ -341,31 +325,47 @@ templates: name: !!python/tuple - a_good_translation-en-de-source+target reference: '' - f231daf4-2277-4a91-8be5-3319fbeb621b: !Template + e99b37c7-6d10-4a14-a462-8b91f2ef0f7e: !Template answer_choices: null - id: f231daf4-2277-4a91-8be5-3319fbeb621b - jinja: 'What is the English translation of : {{translation["de"]}} + id: e99b37c7-6d10-4a14-a462-8b91f2ef0f7e + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: GPT-3 paper - ffc5597a-dea3-4530-982a-81b2f4f8ec74: !Template + - version-en-de-source+target + reference: '' + f580b5fc-4f9c-4b8e-b979-028475218e72: !Template answer_choices: null - id: ffc5597a-dea3-4530-982a-81b2f4f8ec74 - jinja: '{{translation["en"]}} = German: + id: f580b5fc-4f9c-4b8e-b979-028475218e72 + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 + - version-de-en-source+target + reference: '' + fb2fc0a0-7f82-4bfb-a4a8-d38ed93400aa: !Template + answer_choices: null + id: fb2fc0a0-7f82-4bfb-a4a8-d38ed93400aa + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' diff --git a/promptsource/templates/wmt16/fi-en/templates.yaml b/promptsource/templates/wmt16/fi-en/templates.yaml index 32d675257..4e0abaf76 100644 --- a/promptsource/templates/wmt16/fi-en/templates.yaml +++ b/promptsource/templates/wmt16/fi-en/templates.yaml @@ -1,95 +1,94 @@ dataset: wmt16 subset: fi-en templates: - 149671ed-74c4-46eb-a89c-fd1868585d84: !Template + 013f3c8b-7d3c-4729-b5c2-4d408ef53df3: !Template answer_choices: null - id: 149671ed-74c4-46eb-a89c-fd1868585d84 - jinja: '{{translation["en"]}} = Finnish: + id: 013f3c8b-7d3c-4729-b5c2-4d408ef53df3 + jinja: 'What is the Finnish translation of: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 18d30167-1063-42e5-9ec7-5d640e1f1b12: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 11c13b7b-ca3b-492d-ac9c-047046969ed7: !Template answer_choices: null - id: 18d30167-1063-42e5-9ec7-5d640e1f1b12 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: - - ||| {{translation["fi"]}}' + id: 11c13b7b-ca3b-492d-ac9c-047046969ed7 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target + - a_good_translation-fi-en-source+target reference: '' - 1a1dd83a-cbf4-4275-af5e-ea820a00e7e0: !Template + 21ea1113-61f4-4944-a844-162345673f80: !Template answer_choices: null - id: 1a1dd83a-cbf4-4275-af5e-ea820a00e7e0 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: 21ea1113-61f4-4944-a844-162345673f80 + jinja: 'Translate this from English into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-source+target - reference: GPT-3 paper - 1a6db208-8811-485d-bd07-2e8c137e3b42: !Template + - translate-en-fi-source+starget + reference: '' + 2913d203-ac99-4b85-a0d9-57157ab002e9: !Template answer_choices: null - id: 1a6db208-8811-485d-bd07-2e8c137e3b42 - jinja: 'Translate into English: {{translation["fi"]}} + id: 2913d203-ac99-4b85-a0d9-57157ab002e9 + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - translate_as_en-fi-source+target reference: '' - 34fb63e0-3923-415a-86f2-159a8278a98d: !Template + 360be0fb-aef3-4be7-974c-0d1e68f678ec: !Template answer_choices: null - id: 34fb63e0-3923-415a-86f2-159a8278a98d - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: 360be0fb-aef3-4be7-974c-0d1e68f678ec + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-source+target - reference: GPT 3 paper - 39a2e39b-22a2-45b5-b919-88ec767ee423: !Template + - a_good_translation-fi-en-target + reference: '' + 40fc37c3-33dd-45a9-96ff-a4a1a77f8008: !Template answer_choices: null - id: 39a2e39b-22a2-45b5-b919-88ec767ee423 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 40fc37c3-33dd-45a9-96ff-a4a1a77f8008 + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - version-en-fi-target reference: '' - 3d238ee8-502f-42c7-a20c-714caa6a95dc: !Template + 42ad8580-10ba-4df3-8efb-5c531db58279: !Template answer_choices: null - id: 3d238ee8-502f-42c7-a20c-714caa6a95dc + id: 42ad8580-10ba-4df3-8efb-5c531db58279 jinja: ' {{translation["en"]}} translates into Finnish as: ||| {{translation["fi"]}}' @@ -101,138 +100,137 @@ templates: name: !!python/tuple - translate_as_en-fi-target reference: '' - 4192a66e-91e7-4c1a-ba77-6e21716848e6: !Template + 48593d42-382b-4ee0-9db9-a9323e7f20f1: !Template answer_choices: null - id: 4192a66e-91e7-4c1a-ba77-6e21716848e6 - jinja: 'Finnish: {{translation["fi"]}} = English: + id: 48593d42-382b-4ee0-9db9-a9323e7f20f1 + jinja: 'English: {{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 5716cdc9-23bc-4d73-9652-e09868f6f21a: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 500e6c30-7467-4ba7-8444-e3760fd624ad: !Template answer_choices: null - id: 5716cdc9-23bc-4d73-9652-e09868f6f21a - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: 500e6c30-7467-4ba7-8444-e3760fd624ad + jinja: 'Translate this into Finnish: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - translate-en-fi-target reference: '' - 5c474ea2-c34c-4049-9a4e-97ef025e4085: !Template + 746854b6-6ad3-494b-818b-33424c4f5e0f: !Template answer_choices: null - id: 5c474ea2-c34c-4049-9a4e-97ef025e4085 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: 746854b6-6ad3-494b-818b-33424c4f5e0f + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - how_to_say-fi-en-target reference: '' - 6a226e13-cd90-42aa-b7b2-e98d0c542513: !Template + 83e27a76-96d8-41f9-a128-93a43a66c5ab: !Template answer_choices: null - id: 6a226e13-cd90-42aa-b7b2-e98d0c542513 - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: 83e27a76-96d8-41f9-a128-93a43a66c5ab + jinja: '{{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target - reference: GPT-3 paper - 6de62fd5-737f-4c86-bb36-fe9d741dc992: !Template + - translate_as_fi-en-target + reference: '' + 8f218d6c-ed80-421a-8714-b2bf8195982f: !Template answer_choices: null - id: 6de62fd5-737f-4c86-bb36-fe9d741dc992 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: - - ||| {{translation["fi"]}}' + id: 8f218d6c-ed80-421a-8714-b2bf8195982f + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - a_good_translation-en-fi-target reference: '' - 8275e339-4924-4525-a6a3-c8855d02ec7a: !Template + 9728f940-4009-4772-bed0-67afe20b94c7: !Template answer_choices: null - id: 8275e339-4924-4525-a6a3-c8855d02ec7a - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 9728f940-4009-4772-bed0-67afe20b94c7 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-source+starget - reference: '' - 8eaa9792-9b2e-4350-a362-9da0aca71110: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 9aa17975-412f-4f0c-9aab-0a2dfcc1289d: !Template answer_choices: null - id: 8eaa9792-9b2e-4350-a362-9da0aca71110 - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 9aa17975-412f-4f0c-9aab-0a2dfcc1289d + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: GPT-3 paper - 904df1cb-a22f-4dc3-a4a1-65a0255f6a36: !Template + - a_good_translation-en-fi-source+target + reference: '' + a35aecd2-f51f-4975-8653-6b983d1bb641: !Template answer_choices: null - id: 904df1cb-a22f-4dc3-a4a1-65a0255f6a36 - jinja: 'English: {{translation["en"]}} = Finnish: + id: a35aecd2-f51f-4975-8653-6b983d1bb641 + jinja: 'Translate this into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 94309855-afd5-4235-8731-b8c6959d0c27: !Template + - translate-fi-en-target + reference: '' + a79f0128-f6fb-49b3-a27d-7883c17604d9: !Template answer_choices: null - id: 94309855-afd5-4235-8731-b8c6959d0c27 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: a79f0128-f6fb-49b3-a27d-7883c17604d9 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target + - version-fi-en-target reference: '' - 9a096544-5a59-4f74-8b0f-760aab4123b1: !Template + acb870da-f2d4-4b8c-b445-05424f90a81e: !Template answer_choices: null - id: 9a096544-5a59-4f74-8b0f-760aab4123b1 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: acb870da-f2d4-4b8c-b445-05424f90a81e + jinja: '{{translation["en"]}} = Finnish: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -241,53 +239,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target - reference: '' - a09933b1-c90d-4c4b-beb8-f8bb3492d70e: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b1676638-561d-4bbf-9f44-4626b74a468e: !Template answer_choices: null - id: a09933b1-c90d-4c4b-beb8-f8bb3492d70e - jinja: 'How do you say {{translation["fi"]}} in English? + id: b1676638-561d-4bbf-9f44-4626b74a468e + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target - reference: '' - b10cd627-6233-4035-b4e2-b0b94bf84334: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + b9ab9732-c8ff-479c-9a4d-d70b6776c2a1: !Template answer_choices: null - id: b10cd627-6233-4035-b4e2-b0b94bf84334 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: b9ab9732-c8ff-479c-9a4d-d70b6776c2a1 + jinja: 'What is the English translation of : {{translation["fi"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target - reference: '' - b16016c7-2050-477a-aa2d-2866d203d045: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + c196ea94-5087-4312-9711-75e3933a2f00: !Template answer_choices: null - id: b16016c7-2050-477a-aa2d-2866d203d045 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: c196ea94-5087-4312-9711-75e3933a2f00 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - version-fi-en-source+target reference: '' - b250a904-f24b-41a2-b38e-93eca25bd103: !Template + c365ebd2-2981-44ae-8291-b36550bffecb: !Template answer_choices: null - id: b250a904-f24b-41a2-b38e-93eca25bd103 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: c365ebd2-2981-44ae-8291-b36550bffecb + jinja: 'Translate this from Finnish into English: {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -298,67 +298,67 @@ templates: name: !!python/tuple - translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - b28ba135-022b-4ab9-9c99-79e1e286fe4a: !Template + c9d0c7a8-89ee-48b0-9c24-6dbb1dd8025f: !Template answer_choices: null - id: b28ba135-022b-4ab9-9c99-79e1e286fe4a - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: c9d0c7a8-89ee-48b0-9c24-6dbb1dd8025f + jinja: 'Finnish: {{translation["fi"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - translate_as_fi-en-source+target reference: '' - b435a4aa-d75d-4ecb-a8ee-a6c9ae589171: !Template + d83bbd3f-e0d3-41a5-a16e-62f2995f485b: !Template answer_choices: null - id: b435a4aa-d75d-4ecb-a8ee-a6c9ae589171 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: d83bbd3f-e0d3-41a5-a16e-62f2995f485b + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target - reference: '' - bd99a7cf-df50-4355-adeb-3573fd935797: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + d9dfea8a-2f5a-47f6-90e0-7c6875c5f027: !Template answer_choices: null - id: bd99a7cf-df50-4355-adeb-3573fd935797 - jinja: '{{translation["fi"]}} = English: + id: d9dfea8a-2f5a-47f6-90e0-7c6875c5f027 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d03d3586-e7a9-4ff8-b779-fc1504260505: !Template + - how_to_say-en-fi-target + reference: '' + efacbe75-c4ff-4dd9-bbfa-80f4bee92f60: !Template answer_choices: null - id: d03d3586-e7a9-4ff8-b779-fc1504260505 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: efacbe75-c4ff-4dd9-bbfa-80f4bee92f60 + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - version-en-fi-source+target reference: '' - e7edf827-72d4-4e6e-8947-1cf28932715b: !Template + f8b963ef-3189-40d2-9eee-75b7285c3fdd: !Template answer_choices: null - id: e7edf827-72d4-4e6e-8947-1cf28932715b - jinja: '{{translation["fi"]}} translates into English as: + id: f8b963ef-3189-40d2-9eee-75b7285c3fdd + jinja: '{{translation["fi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -367,5 +367,5 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target - reference: '' + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/ro-en/templates.yaml b/promptsource/templates/wmt16/ro-en/templates.yaml index acfbc141b..388cf9144 100644 --- a/promptsource/templates/wmt16/ro-en/templates.yaml +++ b/promptsource/templates/wmt16/ro-en/templates.yaml @@ -1,38 +1,37 @@ dataset: wmt16 subset: ro-en templates: - 05b08ca5-11eb-4eed-a634-d8e146cbffbd: !Template + 02c16dfe-b491-426d-a738-9f8cd6350032: !Template answer_choices: null - id: 05b08ca5-11eb-4eed-a634-d8e146cbffbd - jinja: 'Given the following passage: {{translation["ro"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: 02c16dfe-b491-426d-a738-9f8cd6350032 + jinja: 'Given the following source text in Romanian: {{translation["ro"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ro-en-target + - a_good_translation-ro-en-source+target reference: '' - 0ae186cb-de3f-47dc-b1a6-d787844a79fd: !Template + 10043255-0631-41e6-9f93-20f864d69eeb: !Template answer_choices: null - id: 0ae186cb-de3f-47dc-b1a6-d787844a79fd - jinja: 'Given the following passage: {{translation["en"]}} , a good Romanian translation - is: ||| {{translation["ro"]}}' + id: 10043255-0631-41e6-9f93-20f864d69eeb + jinja: 'Romanian: {{translation["ro"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ro-target - reference: '' - 0e7422d6-244f-40d9-be09-6d9248affa6c: !Template + - xglm-ro-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 113e022f-9d70-40f1-855f-024c5badad20: !Template answer_choices: null - id: 0e7422d6-244f-40d9-be09-6d9248affa6c - jinja: 'Translate Romanian into English: {{translation["ro"]}} + id: 113e022f-9d70-40f1-855f-024c5badad20 + jinja: 'Translate this from Romanian into English: {{translation["ro"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -43,39 +42,39 @@ templates: name: !!python/tuple - translate-ro-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - 3743872c-f834-4c4d-a32c-1ad035f8b7cd: !Template + 1519ffc7-7a0b-4a04-b626-7265f93ad09f: !Template answer_choices: null - id: 3743872c-f834-4c4d-a32c-1ad035f8b7cd - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Romanian translation is: + id: 1519ffc7-7a0b-4a04-b626-7265f93ad09f + jinja: 'If the original version says: {{translation["en"]}}; then the Romanian + version should say: - ||| {{translation["ro"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ro-source+target + - version-en-ro-target reference: '' - 3ed2a5f2-c6ea-4b18-a45e-045eafe0592e: !Template + 1709e27a-1069-4124-9ab9-fb571d05cd0b: !Template answer_choices: null - id: 3ed2a5f2-c6ea-4b18-a45e-045eafe0592e - jinja: ' {{translation["en"]}} translates into Romanian as: + id: 1709e27a-1069-4124-9ab9-fb571d05cd0b + jinja: 'Translate this from English into Romanian: {{translation["en"]}} - ||| {{translation["ro"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ro-target + - translate-en-ro-source+starget reference: '' - 42b4bf2d-1bb5-4669-b041-dc761dfb84eb: !Template + 22ecf1d7-6909-4b18-aac1-42d4104a101e: !Template answer_choices: null - id: 42b4bf2d-1bb5-4669-b041-dc761dfb84eb - jinja: 'Romanian: {{translation["ro"]}} = English: + id: 22ecf1d7-6909-4b18-aac1-42d4104a101e + jinja: '{{translation["ro"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -84,12 +83,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-ro-en-source+target + - xglm-ro-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 465d16b0-1dd0-48d4-a07b-f99b262feeb5: !Template + 26818653-4a13-456a-919e-8e46f4cca807: !Template answer_choices: null - id: 465d16b0-1dd0-48d4-a07b-f99b262feeb5 - jinja: '{{translation["en"]}} = Romanian: + id: 26818653-4a13-456a-919e-8e46f4cca807 + jinja: 'How do you say {{translation["en"]}} in Romanian? ||| {{translation["ro"]}}' metadata: !TemplateMetadata @@ -98,12 +97,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-ro-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4e2fcab2-05fb-4099-a038-9c585b9b4be7: !Template + - how_to_say-en-ro-target + reference: '' + 29397cea-e7fe-46c8-821d-75925174235b: !Template answer_choices: null - id: 4e2fcab2-05fb-4099-a038-9c585b9b4be7 - jinja: 'English: {{translation["en"]}} translates into Romanian as: + id: 29397cea-e7fe-46c8-821d-75925174235b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Romanian translation is: ||| {{translation["ro"]}}' metadata: !TemplateMetadata @@ -112,26 +112,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ro-source+target + - a_good_translation-en-ro-source+target reference: '' - 5bf16978-6b7d-468a-b84f-943f0179a548: !Template + 305a6ff4-5f27-4150-96cb-0f4657065de6: !Template answer_choices: null - id: 5bf16978-6b7d-468a-b84f-943f0179a548 - jinja: 'English: {{translation["en"]}} = Romanian: + id: 305a6ff4-5f27-4150-96cb-0f4657065de6 + jinja: 'What is the English translation of : {{translation["ro"]}} - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ro-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 62e17c10-5d28-4822-88f2-5eaec1d09c9e: !Template + - gpt-3-ro-en-target + reference: GPT-3 paper + 3ea8048e-8c88-47cf-85e1-be83a925edde: !Template answer_choices: null - id: 62e17c10-5d28-4822-88f2-5eaec1d09c9e - jinja: 'What is the Romanian translation of the English sentence: {{translation["en"]}} + id: 3ea8048e-8c88-47cf-85e1-be83a925edde + jinja: 'What is the Romanian translation of: {{translation["en"]}} ||| {{translation["ro"]}}' metadata: !TemplateMetadata @@ -140,27 +140,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ro-source+target - reference: GPT 3 paper - 6eade7af-18af-40b4-8369-7ca24f043914: !Template + - gpt-3-en-ro-target + reference: GPT-3 paper + 53283c05-89bd-40af-b68a-588e26bcabf4: !Template answer_choices: null - id: 6eade7af-18af-40b4-8369-7ca24f043914 - jinja: 'If the English version says: {{translation["en"]}}; then the Romanian - version should say: + id: 53283c05-89bd-40af-b68a-588e26bcabf4 + jinja: 'Romanian: {{translation["ro"]}} translates into English as: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ro-source+target + - translate_as_ro-en-source+target reference: '' - 703343c4-14bb-44b2-aed8-034433d1d251: !Template + 70758f4b-f327-4490-95fa-0998502e83dc: !Template answer_choices: null - id: 703343c4-14bb-44b2-aed8-034433d1d251 - jinja: 'How do you say {{translation["en"]}} in Romanian? + id: 70758f4b-f327-4490-95fa-0998502e83dc + jinja: ' {{translation["en"]}} translates into Romanian as: ||| {{translation["ro"]}}' metadata: !TemplateMetadata @@ -169,97 +168,97 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ro-target + - translate_as_en-ro-target reference: '' - 8789e689-e914-4a09-9614-d27b38f05528: !Template + 7d6958fb-c088-49f3-a7a0-06f031045413: !Template answer_choices: null - id: 8789e689-e914-4a09-9614-d27b38f05528 - jinja: 'Translate English into Romanian: {{translation["en"]}} + id: 7d6958fb-c088-49f3-a7a0-06f031045413 + jinja: 'Given the following passage: {{translation["ro"]}} , a good English translation + is: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ro-source+starget + - a_good_translation-ro-en-target reference: '' - 981b526d-aad9-497b-8563-c523b640c12e: !Template + 8307a62a-4fe3-4c51-bd6a-39aa0ad1869e: !Template answer_choices: null - id: 981b526d-aad9-497b-8563-c523b640c12e - jinja: 'If the Romanian version says: {{translation["ro"]}}; then the English - version should say: - - ||| {{translation["en"]}}' + id: 8307a62a-4fe3-4c51-bd6a-39aa0ad1869e + jinja: 'Given the following passage: {{translation["en"]}} , a good Romanian translation + is: ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ro-en-source+target + - a_good_translation-en-ro-target reference: '' - 9c5af958-102b-4fc7-a477-b7fad827f07f: !Template + 8391bcb6-6643-4cec-ad86-218820783c56: !Template answer_choices: null - id: 9c5af958-102b-4fc7-a477-b7fad827f07f - jinja: 'Translate into English: {{translation["ro"]}} + id: 8391bcb6-6643-4cec-ad86-218820783c56 + jinja: 'English: {{translation["en"]}} = Romanian: - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ro-en-target - reference: '' - a40eb7aa-fe7e-4fd6-bc2f-dcd295a2e96d: !Template + - xglm-en-ro-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 8e245134-0607-4fa0-abbf-7725cfce1b7c: !Template answer_choices: null - id: a40eb7aa-fe7e-4fd6-bc2f-dcd295a2e96d - jinja: '{{translation["ro"]}} = English: + id: 8e245134-0607-4fa0-abbf-7725cfce1b7c + jinja: 'Translate this into Romanian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ro-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - a4c38371-d526-4783-8213-39df0ec8ff12: !Template + - translate-en-ro-target + reference: '' + 97592b70-84ec-4571-8101-844ee3bffac8: !Template answer_choices: null - id: a4c38371-d526-4783-8213-39df0ec8ff12 - jinja: 'If the original version says: {{translation["en"]}}; then the Romanian - version should say: + id: 97592b70-84ec-4571-8101-844ee3bffac8 + jinja: '{{translation["en"]}} = Romanian: - ||| {{translation["ro"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ro-target - reference: '' - b01b7756-3879-4814-9909-0cd565d172f5: !Template + - xglm-en-ro-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a438d7f1-3421-4c58-a211-2d33c7c393cf: !Template answer_choices: null - id: b01b7756-3879-4814-9909-0cd565d172f5 - jinja: 'Given the following source text in Romanian: {{translation["ro"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: a438d7f1-3421-4c58-a211-2d33c7c393cf + jinja: 'If the Romanian version says: {{translation["ro"]}}; then the English + version should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ro-en-source+target + - version-ro-en-source+target reference: '' - b64b25c0-af3e-4247-bc53-8c40048cd9e7: !Template + a5604743-a810-4ae3-9811-f3906d7fcebb: !Template answer_choices: null - id: b64b25c0-af3e-4247-bc53-8c40048cd9e7 - jinja: 'How do you say {{translation["ro"]}} in English? + id: a5604743-a810-4ae3-9811-f3906d7fcebb + jinja: '{{translation["ro"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -268,104 +267,105 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-ro-en-target + - translate_as_ro-en-target reference: '' - b6920ff2-7442-4d06-ad2a-3dd6d58a53ee: !Template + b7c7bbe1-e873-4a5b-b86b-1689cf814cc9: !Template answer_choices: null - id: b6920ff2-7442-4d06-ad2a-3dd6d58a53ee - jinja: 'If the original version says: {{translation["ro"]}}; then the English - version should say: + id: b7c7bbe1-e873-4a5b-b86b-1689cf814cc9 + jinja: 'Translate this into English: {{translation["ro"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ro-en-target + - translate-ro-en-target reference: '' - bcea9f08-def5-4711-8a91-7f5f2191e4cb: !Template + d13385f1-2dfa-486d-9d64-8fcca0c8f449: !Template answer_choices: null - id: bcea9f08-def5-4711-8a91-7f5f2191e4cb - jinja: '{{translation["ro"]}} translates into English as: + id: d13385f1-2dfa-486d-9d64-8fcca0c8f449 + jinja: 'What is the Romanian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ro-en-target - reference: '' - d3eb0c1d-242a-401e-8e0c-97ad9ef6bf06: !Template + - gpt-3-en-ro-source+target + reference: GPT 3 paper + e3b3b252-d557-4640-b21a-df27fe7a03ce: !Template answer_choices: null - id: d3eb0c1d-242a-401e-8e0c-97ad9ef6bf06 - jinja: 'Romanian: {{translation["ro"]}} translates into English as: + id: e3b3b252-d557-4640-b21a-df27fe7a03ce + jinja: 'What is the English translation of the Romanian sentence: {{translation["ro"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ro-en-source+target - reference: '' - dbd449cd-bc4f-44e0-9634-a69ecf5a5c1b: !Template + - gpt-3-ro-en-source+target + reference: GPT-3 paper + e83b3367-9a91-41db-a161-ddfdc30cfca1: !Template answer_choices: null - id: dbd449cd-bc4f-44e0-9634-a69ecf5a5c1b - jinja: 'What is the English translation of : {{translation["ro"]}} + id: e83b3367-9a91-41db-a161-ddfdc30cfca1 + jinja: 'English: {{translation["en"]}} translates into Romanian as: - ||| {{translation["en"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ro-en-target - reference: GPT-3 paper - f4d2dea9-6b81-4709-a2da-fcd21bde5107: !Template + - translate_as_en-ro-source+target + reference: '' + f317bb24-5828-43d8-8f8f-74b193f96396: !Template answer_choices: null - id: f4d2dea9-6b81-4709-a2da-fcd21bde5107 - jinja: 'Translate into Romanian: {{translation["en"]}} + id: f317bb24-5828-43d8-8f8f-74b193f96396 + jinja: 'If the English version says: {{translation["en"]}}; then the Romanian + version should say: - ||| {{translation["ro"]}}' + ||| {{translation["ro"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ro-target + - version-en-ro-source+target reference: '' - fc553e43-27d2-4ed5-8c95-743972d91a47: !Template + f70cde91-976b-44d3-a77b-284e26aba970: !Template answer_choices: null - id: fc553e43-27d2-4ed5-8c95-743972d91a47 - jinja: 'What is the Romanian translation of: {{translation["en"]}} + id: f70cde91-976b-44d3-a77b-284e26aba970 + jinja: 'If the original version says: {{translation["ro"]}}; then the English + version should say: - ||| {{translation["ro"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ro-target - reference: GPT-3 paper - fc9a94ab-3355-4af0-9960-387303ab513b: !Template + - version-ro-en-target + reference: '' + fd8c924a-93bb-480f-be25-94ac888b7e30: !Template answer_choices: null - id: fc9a94ab-3355-4af0-9960-387303ab513b - jinja: 'What is the English translation of the Romanian sentence: {{translation["ro"]}} + id: fd8c924a-93bb-480f-be25-94ac888b7e30 + jinja: 'How do you say {{translation["ro"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ro-en-source+target - reference: GPT-3 paper + - how_to_say-ro-en-target + reference: '' diff --git a/promptsource/templates/wmt16/ru-en/templates.yaml b/promptsource/templates/wmt16/ru-en/templates.yaml index 78418f0b5..3dc589867 100644 --- a/promptsource/templates/wmt16/ru-en/templates.yaml +++ b/promptsource/templates/wmt16/ru-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt16 subset: ru-en templates: - 03d02f14-f91d-421e-8fae-a91fc908dbab: !Template + 044ecf25-9893-4416-a63e-f88f78ad53d0: !Template answer_choices: null - id: 03d02f14-f91d-421e-8fae-a91fc908dbab + id: 044ecf25-9893-4416-a63e-f88f78ad53d0 jinja: '{{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' @@ -15,24 +15,10 @@ templates: name: !!python/tuple - translate_as_ru-en-target reference: '' - 05d68683-b85b-43e4-9b61-7f293c736ea0: !Template + 09094227-68fe-4462-9477-11683a34452f: !Template answer_choices: null - id: 05d68683-b85b-43e4-9b61-7f293c736ea0 - jinja: 'What is the Russian translation of: {{translation["en"]}} - - ||| {{translation["ru"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - gpt-3-en-ru-target - reference: GPT-3 paper - 1666915a-19e0-47a1-a935-52645db05501: !Template - answer_choices: null - id: 1666915a-19e0-47a1-a935-52645db05501 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 09094227-68fe-4462-9477-11683a34452f + jinja: 'Translate this from Russian into English: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -43,25 +29,24 @@ templates: name: !!python/tuple - translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - 19c0c34c-25a8-481a-8c7e-74d6032a1e26: !Template + 094e53f2-faec-40df-bbce-43baf9aab2bb: !Template answer_choices: null - id: 19c0c34c-25a8-481a-8c7e-74d6032a1e26 - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 094e53f2-faec-40df-bbce-43baf9aab2bb + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' - 1c3f4328-4b9c-45a8-a62b-97839c9b5429: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 0beb1dab-caed-4fdf-b248-aa8d1013c09a: !Template answer_choices: null - id: 1c3f4328-4b9c-45a8-a62b-97839c9b5429 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 0beb1dab-caed-4fdf-b248-aa8d1013c09a + jinja: 'Translate this from English into Russian: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -72,38 +57,38 @@ templates: name: !!python/tuple - translate-en-ru-source+starget reference: '' - 28128ce4-3426-4fdc-84c7-9861387933d3: !Template + 232d5901-cbeb-4db3-b714-56bd2afd30a1: !Template answer_choices: null - id: 28128ce4-3426-4fdc-84c7-9861387933d3 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 232d5901-cbeb-4db3-b714-56bd2afd30a1 + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-source+target - reference: GPT-3 paper - 2e10725a-66ab-41d7-af3b-23013e99d4f7: !Template + - version-ru-en-source+target + reference: '' + 232f4031-3ef7-4929-9d7c-f248ac0aef6c: !Template answer_choices: null - id: 2e10725a-66ab-41d7-af3b-23013e99d4f7 - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 232f4031-3ef7-4929-9d7c-f248ac0aef6c + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target - reference: '' - 36d643f8-5e9f-4536-ad88-fb72c317b576: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 31503446-f6ce-48e4-8f12-4f7a9ba34e38: !Template answer_choices: null - id: 36d643f8-5e9f-4536-ad88-fb72c317b576 + id: 31503446-f6ce-48e4-8f12-4f7a9ba34e38 jinja: 'Russian: {{translation["ru"]}} = English: ||| {{translation["en"]}}' @@ -115,10 +100,11 @@ templates: name: !!python/tuple - xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 3cbae036-ede7-47dd-b3bb-eeecaaaf47c0: !Template + 34fcf6e1-326b-48db-81da-b8a545991c6b: !Template answer_choices: null - id: 3cbae036-ede7-47dd-b3bb-eeecaaaf47c0 - jinja: '{{translation["en"]}} = Russian: + id: 34fcf6e1-326b-48db-81da-b8a545991c6b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -127,27 +113,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 65300887-6919-4804-9594-c7e9880fb62c: !Template + - a_good_translation-en-ru-source+target + reference: '' + 471fb738-1a7d-4f2c-a65b-71a78dc8f8cf: !Template answer_choices: null - id: 65300887-6919-4804-9594-c7e9880fb62c - jinja: 'How do you say {{translation["ru"]}} in English? + id: 471fb738-1a7d-4f2c-a65b-71a78dc8f8cf + jinja: 'Translate this into English: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate-ru-en-target reference: '' - 6fde4437-43c8-4e4e-9a64-b961036b8977: !Template + 59cffc19-7a33-44cb-a8a6-a29f61ae000e: !Template answer_choices: null - id: 6fde4437-43c8-4e4e-9a64-b961036b8977 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 59cffc19-7a33-44cb-a8a6-a29f61ae000e + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -156,28 +142,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - version-en-ru-source+target reference: '' - 76eea0de-6fd9-485f-9ebf-bc4a51e4d600: !Template + 5b4822ef-15c1-446e-85c7-b6665c96feb3: !Template answer_choices: null - id: 76eea0de-6fd9-485f-9ebf-bc4a51e4d600 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 5b4822ef-15c1-446e-85c7-b6665c96feb3 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - version-ru-en-target reference: '' - 773390d4-3602-46e5-b76a-c09ac8e957ca: !Template + 69bd54a9-2ec4-4c30-b951-4ce9875e4135: !Template answer_choices: null - id: 773390d4-3602-46e5-b76a-c09ac8e957ca - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 69bd54a9-2ec4-4c30-b951-4ce9875e4135 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -186,79 +171,80 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - translate_as_en-ru-source+target reference: '' - 78c8a6cc-24b3-4c2c-a9ce-daf57b1c712c: !Template + 6f97021a-7679-4afb-b60c-454c5951b3ca: !Template answer_choices: null - id: 78c8a6cc-24b3-4c2c-a9ce-daf57b1c712c - jinja: 'What is the English translation of : {{translation["ru"]}} - - ||| {{translation["en"]}}' + id: 6f97021a-7679-4afb-b60c-454c5951b3ca + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - 7cf6777c-e32c-4cd9-b794-fc86e29e8083: !Template + - a_good_translation-en-ru-target + reference: '' + 6fe335f8-c91a-41b6-8a31-1cfe03e19f7b: !Template answer_choices: null - id: 7cf6777c-e32c-4cd9-b794-fc86e29e8083 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 6fe335f8-c91a-41b6-8a31-1cfe03e19f7b + jinja: '{{translation["en"]}} = Russian: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target - reference: '' - 90c0449c-8364-4663-bee9-8217770fcea0: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 87c91692-c5d2-46d7-b700-4238f7156e05: !Template answer_choices: null - id: 90c0449c-8364-4663-bee9-8217770fcea0 - jinja: 'Translate into Russian: {{translation["en"]}} + id: 87c91692-c5d2-46d7-b700-4238f7156e05 + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target - reference: '' - a0d1d119-df3c-437a-8940-a84fa73b9eeb: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 90adb529-bc50-42d1-8229-bdfaeb52cf68: !Template answer_choices: null - id: a0d1d119-df3c-437a-8940-a84fa73b9eeb - jinja: '{{translation["ru"]}} = English: + id: 90adb529-bc50-42d1-8229-bdfaeb52cf68 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - a8afdc6f-1b33-4773-8bda-37a01030cef2: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 9aa8bab8-460c-44a4-9367-60d2d32be39d: !Template answer_choices: null - id: a8afdc6f-1b33-4773-8bda-37a01030cef2 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 9aa8bab8-460c-44a4-9367-60d2d32be39d + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - how_to_say-ru-en-target reference: '' - aa4db5da-1829-40c2-9ca4-634f0279b579: !Template + 9c49e553-4823-46b8-9bfe-6b29761dc88f: !Template answer_choices: null - id: aa4db5da-1829-40c2-9ca4-634f0279b579 + id: 9c49e553-4823-46b8-9bfe-6b29761dc88f jinja: 'Russian: {{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' @@ -270,53 +256,67 @@ templates: name: !!python/tuple - translate_as_ru-en-source+target reference: '' - b191cdab-98c6-44cf-a864-1fa8a73c0f78: !Template + a55c925c-d1e0-4d86-95cf-e0d9477d1e86: !Template answer_choices: null - id: b191cdab-98c6-44cf-a864-1fa8a73c0f78 - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: a55c925c-d1e0-4d86-95cf-e0d9477d1e86 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - a_good_translation-ru-en-target reference: '' - c94ef825-4ad3-4128-9cba-29581906ccdf: !Template + aed6444f-0558-4234-8b2a-6885a5906491: !Template answer_choices: null - id: c94ef825-4ad3-4128-9cba-29581906ccdf - jinja: ' {{translation["en"]}} translates into Russian as: + id: aed6444f-0558-4234-8b2a-6885a5906491 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - gpt-3-ru-en-source+target + reference: GPT-3 paper + b02c260c-50ce-40d6-b9eb-8f8c69285cf6: !Template + answer_choices: null + id: b02c260c-50ce-40d6-b9eb-8f8c69285cf6 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target reference: '' - e46bf114-cad8-4e1f-a5bc-230fc7da194e: !Template + b56e3f29-475c-4b2a-ab2b-1ca9de77a538: !Template answer_choices: null - id: e46bf114-cad8-4e1f-a5bc-230fc7da194e - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: b56e3f29-475c-4b2a-ab2b-1ca9de77a538 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - ef0bdff5-c6d9-4fdd-8269-d67bbbebedd6: !Template + - version-en-ru-target + reference: '' + b844d53a-20dd-4430-b47a-cfd759a7cafb: !Template answer_choices: null - id: ef0bdff5-c6d9-4fdd-8269-d67bbbebedd6 - jinja: 'English: {{translation["en"]}} = Russian: + id: b844d53a-20dd-4430-b47a-cfd759a7cafb + jinja: ' {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -325,11 +325,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - f0c55f7b-9021-4811-8966-000e2654cd39: !Template + - translate_as_en-ru-target + reference: '' + e2c7cce8-1955-4fea-9a9a-9e728d9c37f9: !Template answer_choices: null - id: f0c55f7b-9021-4811-8966-000e2654cd39 + id: e2c7cce8-1955-4fea-9a9a-9e728d9c37f9 jinja: 'How do you say {{translation["en"]}} in Russian? ||| {{translation["ru"]}}' @@ -341,31 +341,31 @@ templates: name: !!python/tuple - how_to_say-en-ru-target reference: '' - f42e0011-a338-47d6-971f-57cf1de83a82: !Template + fcf79f35-fd25-4281-ab6e-d1619d2477a7: !Template answer_choices: null - id: f42e0011-a338-47d6-971f-57cf1de83a82 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: fcf79f35-fd25-4281-ab6e-d1619d2477a7 + jinja: 'Translate this into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - translate-en-ru-target reference: '' - f9c2e846-0274-4933-bd9f-615e572e376d: !Template + febf5e43-6af8-4d1e-8c20-3e454239c420: !Template answer_choices: null - id: f9c2e846-0274-4933-bd9f-615e572e376d - jinja: 'Translate into English: {{translation["ru"]}} + id: febf5e43-6af8-4d1e-8c20-3e454239c420 + jinja: 'English: {{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target - reference: '' + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/tr-en/templates.yaml b/promptsource/templates/wmt16/tr-en/templates.yaml index 4a8b2b6ae..cf149420c 100644 --- a/promptsource/templates/wmt16/tr-en/templates.yaml +++ b/promptsource/templates/wmt16/tr-en/templates.yaml @@ -1,166 +1,165 @@ dataset: wmt16 subset: tr-en templates: - 03be6519-f6d4-4fb9-81bf-a6c32e54f624: !Template + 00e574df-683b-485d-8b03-8d325294444f: !Template answer_choices: null - id: 03be6519-f6d4-4fb9-81bf-a6c32e54f624 - jinja: 'What is the English translation of : {{translation["tr"]}} + id: 00e574df-683b-485d-8b03-8d325294444f + jinja: '{{translation["en"]}} = Turkish: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target - reference: GPT-3 paper - 07cfea8a-6d20-4719-a4ba-2ef8af3c416f: !Template + - xglm-en-tr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 0a3d5810-f12a-4bf4-8f73-796445516308: !Template answer_choices: null - id: 07cfea8a-6d20-4719-a4ba-2ef8af3c416f - jinja: 'How do you say {{translation["en"]}} in Turkish? - - ||| {{translation["tr"]}}' + id: 0a3d5810-f12a-4bf4-8f73-796445516308 + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - a_good_translation-en-tr-target reference: '' - 0905d853-aa09-4e6b-a1ec-87a61bfbaabf: !Template + 2798539b-6159-4cfe-ad24-26adcd0de104: !Template answer_choices: null - id: 0905d853-aa09-4e6b-a1ec-87a61bfbaabf - jinja: '{{translation["tr"]}} translates into English as: + id: 2798539b-6159-4cfe-ad24-26adcd0de104 + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-target - reference: '' - 0fe7d6a7-36d9-47ab-932d-b6a260f35893: !Template + - gpt-3-tr-en-source+target + reference: GPT-3 paper + 32814e82-f25c-4421-827a-801c37507852: !Template answer_choices: null - id: 0fe7d6a7-36d9-47ab-932d-b6a260f35893 - jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version - should say: + id: 32814e82-f25c-4421-827a-801c37507852 + jinja: 'Translate this into English: {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-source+target + - translate-tr-en-target reference: '' - 161d9236-0bf4-48f3-896c-7814b8b516a3: !Template + 3d717faf-57ea-4912-8441-93e8227fbe66: !Template answer_choices: null - id: 161d9236-0bf4-48f3-896c-7814b8b516a3 - jinja: 'Translate into Turkish: {{translation["en"]}} + id: 3d717faf-57ea-4912-8441-93e8227fbe66 + jinja: 'English: {{translation["en"]}} = Turkish: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-target - reference: '' - 20e73f16-c5a1-4309-99a8-1f814c071b94: !Template + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 3d8a7fef-1a66-499c-83aa-2729372dc315: !Template answer_choices: null - id: 20e73f16-c5a1-4309-99a8-1f814c071b94 - jinja: 'English: {{translation["en"]}} translates into Turkish as: + id: 3d8a7fef-1a66-499c-83aa-2729372dc315 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target + - version-en-tr-target reference: '' - 2ede56b7-1bc4-4020-9967-5c5cab76e969: !Template + 4ecce376-edea-4750-82ed-d47227133584: !Template answer_choices: null - id: 2ede56b7-1bc4-4020-9967-5c5cab76e969 - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: 4ecce376-edea-4750-82ed-d47227133584 + jinja: 'How do you say {{translation["tr"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target + - how_to_say-tr-en-target reference: '' - 314ad496-6be3-47db-98bf-69c8552d32e7: !Template + 529393ed-0962-43de-8031-422e8cc8cdd9: !Template answer_choices: null - id: 314ad496-6be3-47db-98bf-69c8552d32e7 - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: 529393ed-0962-43de-8031-422e8cc8cdd9 + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-source+starget + - a_good_translation-tr-en-target reference: '' - 3a2e1972-6f3c-4f85-8bc7-e4bd238ad974: !Template + 5d439e41-e34b-4dad-b883-0228054dfc75: !Template answer_choices: null - id: 3a2e1972-6f3c-4f85-8bc7-e4bd238ad974 - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: 5d439e41-e34b-4dad-b883-0228054dfc75 + jinja: 'Translate this from English into Turkish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 42104024-1eba-41b1-abc5-09377e8b0ed9: !Template + - translate-en-tr-source+starget + reference: '' + 81fd39b4-471b-46fc-9171-73b0ebfaeff3: !Template answer_choices: null - id: 42104024-1eba-41b1-abc5-09377e8b0ed9 - jinja: 'Turkish: {{translation["tr"]}} = English: + id: 81fd39b4-471b-46fc-9171-73b0ebfaeff3 + jinja: 'English: {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4461db59-9557-4aef-b030-aedcfbe3e1b5: !Template + - translate_as_en-tr-source+target + reference: '' + 81ffb065-63b0-42d0-abc7-d0b4af2eae6b: !Template answer_choices: null - id: 4461db59-9557-4aef-b030-aedcfbe3e1b5 - jinja: '{{translation["tr"]}} = English: + id: 81ffb065-63b0-42d0-abc7-d0b4af2eae6b + jinja: 'What is the Turkish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 45077a08-85e6-4986-a0e0-439a9bdfdde0: !Template + - gpt-3-en-tr-target + reference: GPT-3 paper + 820db088-1fb4-4cab-b47b-447b8958b108: !Template answer_choices: null - id: 45077a08-85e6-4986-a0e0-439a9bdfdde0 - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: 820db088-1fb4-4cab-b47b-447b8958b108 + jinja: 'Turkish: {{translation["tr"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -169,152 +168,155 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target - reference: '' - 45569b51-8855-4a84-9d01-7bd38861e88a: !Template + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 85e5c442-bd52-41e1-810f-039c9d30778f: !Template answer_choices: null - id: 45569b51-8855-4a84-9d01-7bd38861e88a - jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 85e5c442-bd52-41e1-810f-039c9d30778f + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-source+target - reference: '' - 59fd0965-c26d-49d4-bb0d-028e7299d9f4: !Template + - gpt-3-en-tr-source+target + reference: GPT 3 paper + 936cef42-229d-40f1-8c1c-1c077f2e6e48: !Template answer_choices: null - id: 59fd0965-c26d-49d4-bb0d-028e7299d9f4 - jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version - should say: + id: 936cef42-229d-40f1-8c1c-1c077f2e6e48 + jinja: ' {{translation["en"]}} translates into Turkish as: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-source+target + - translate_as_en-tr-target reference: '' - 5c3f1006-96be-4b01-8585-6982befd3b4c: !Template + 9eb17600-3717-44c6-90e6-103bd40e16d8: !Template answer_choices: null - id: 5c3f1006-96be-4b01-8585-6982befd3b4c - jinja: ' {{translation["en"]}} translates into Turkish as: + id: 9eb17600-3717-44c6-90e6-103bd40e16d8 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target + - version-tr-en-target reference: '' - 68d36e14-5f6e-4b9a-8a1a-373d56ab58e3: !Template + a028ed7a-482b-4ac9-9339-a888c255e09f: !Template answer_choices: null - id: 68d36e14-5f6e-4b9a-8a1a-373d56ab58e3 - jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation - is: + id: a028ed7a-482b-4ac9-9339-a888c255e09f + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-target + - a_good_translation-en-tr-source+target reference: '' - 7151e15d-1107-4ab3-b94a-98b0600546a4: !Template + a335c2d1-09a5-43fc-91c5-62de6fdde9f1: !Template answer_choices: null - id: 7151e15d-1107-4ab3-b94a-98b0600546a4 - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + id: a335c2d1-09a5-43fc-91c5-62de6fdde9f1 + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-source+target - reference: GPT 3 paper - 9b22fa04-287e-457d-a530-60bfef922b5b: !Template + - version-en-tr-source+target + reference: '' + aeae66d6-b0b6-471b-af3c-51a9d3100b52: !Template answer_choices: null - id: 9b22fa04-287e-457d-a530-60bfef922b5b - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: aeae66d6-b0b6-471b-af3c-51a9d3100b52 + jinja: '{{translation["tr"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-source+target - reference: GPT-3 paper - 9de9346b-8c09-47be-8154-c2c1715e054e: !Template + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b3fe56dc-1a02-4b1b-a0bc-58a4e4e3118e: !Template answer_choices: null - id: 9de9346b-8c09-47be-8154-c2c1715e054e - jinja: '{{translation["en"]}} = Turkish: + id: b3fe56dc-1a02-4b1b-a0bc-58a4e4e3118e + jinja: 'Translate this from Turkish into English: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c01ec82b-9c70-49b4-8244-ddd4ef89e963: !Template + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + c7f4214d-6581-4078-99c1-50eba4b06f97: !Template answer_choices: null - id: c01ec82b-9c70-49b4-8244-ddd4ef89e963 - jinja: 'English: {{translation["en"]}} = Turkish: + id: c7f4214d-6581-4078-99c1-50eba4b06f97 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - ca5fa23d-8591-4d37-8a79-24103a52af45: !Template + - translate_as_tr-en-source+target + reference: '' + cfbad6dc-82f1-4f07-be6b-71e694d479a4: !Template answer_choices: null - id: ca5fa23d-8591-4d37-8a79-24103a52af45 - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: cfbad6dc-82f1-4f07-be6b-71e694d479a4 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target + - version-tr-en-source+target reference: '' - d7e68185-0fe7-41f1-b66b-9ad3e089a3fa: !Template + d37ce743-96e7-4a0d-8dad-d51567bc3904: !Template answer_choices: null - id: d7e68185-0fe7-41f1-b66b-9ad3e089a3fa - jinja: 'What is the Turkish translation of: {{translation["en"]}} - - ||| {{translation["tr"]}}' + id: d37ce743-96e7-4a0d-8dad-d51567bc3904 + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target - reference: GPT-3 paper - dae5163c-ecaf-4eaa-805e-41a8f312eb05: !Template + - a_good_translation-tr-en-source+target + reference: '' + e48579b1-ca64-4434-a8f7-232c9b08abee: !Template answer_choices: null - id: dae5163c-ecaf-4eaa-805e-41a8f312eb05 - jinja: 'Translate into English: {{translation["tr"]}} + id: e48579b1-ca64-4434-a8f7-232c9b08abee + jinja: 'What is the English translation of : {{translation["tr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -323,12 +325,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-target - reference: '' - e4bfc8ed-3370-46d3-90be-187523d10e09: !Template + - gpt-3-tr-en-target + reference: GPT-3 paper + e7d0083f-4003-4e4f-b78c-737d47dd0822: !Template answer_choices: null - id: e4bfc8ed-3370-46d3-90be-187523d10e09 - jinja: 'How do you say {{translation["tr"]}} in English? + id: e7d0083f-4003-4e4f-b78c-737d47dd0822 + jinja: '{{translation["tr"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -337,13 +339,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-tr-en-target + - translate_as_tr-en-target reference: '' - e7c13937-1ced-4129-8de2-f51136e661c9: !Template + ea16adca-5a2e-4d47-aaa5-73b8c16d7409: !Template answer_choices: null - id: e7c13937-1ced-4129-8de2-f51136e661c9 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: ea16adca-5a2e-4d47-aaa5-73b8c16d7409 + jinja: 'How do you say {{translation["en"]}} in Turkish? ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -352,20 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target + - how_to_say-en-tr-target reference: '' - f46b64ae-bbd4-449e-869c-c6c84a1d9aeb: !Template + fb6a6dba-9d0a-4475-9899-8e9567bbe3be: !Template answer_choices: null - id: f46b64ae-bbd4-449e-869c-c6c84a1d9aeb - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: fb6a6dba-9d0a-4475-9899-8e9567bbe3be + jinja: 'Translate this into Turkish: {{translation["en"]}} - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target + - translate-en-tr-target reference: '' diff --git a/promptsource/templates/wmt17/cs-en/templates.yaml b/promptsource/templates/wmt17/cs-en/templates.yaml index d27ac0988..f9e7cf2e1 100644 --- a/promptsource/templates/wmt17/cs-en/templates.yaml +++ b/promptsource/templates/wmt17/cs-en/templates.yaml @@ -1,167 +1,163 @@ dataset: wmt17 subset: cs-en templates: - 02a34024-126f-4147-92a4-85b919389b09: !Template + 04a07a0f-ac2b-494d-b97d-79c41e3b261d: !Template answer_choices: null - id: 02a34024-126f-4147-92a4-85b919389b09 - jinja: 'Translate into Czech: {{translation["en"]}} + id: 04a07a0f-ac2b-494d-b97d-79c41e3b261d + jinja: 'Translate this from Czech into English: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-target - reference: '' - 032b917b-7e01-4f7b-9826-48025d460e60: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 1504c2ef-f956-42b1-b6d3-58bc82859f8c: !Template answer_choices: null - id: 032b917b-7e01-4f7b-9826-48025d460e60 - jinja: 'Translate into English: {{translation["cs"]}} + id: 1504c2ef-f956-42b1-b6d3-58bc82859f8c + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - translate_as_cs-en-source+target reference: '' - 05c1b0e4-2424-434d-9f40-c3d02c88a73b: !Template + 1b44c745-8efe-4aa3-9a78-b4bde3d0e20b: !Template answer_choices: null - id: 05c1b0e4-2424-434d-9f40-c3d02c88a73b - jinja: 'What is the Czech translation of: {{translation["en"]}} - - ||| {{translation["cs"]}}' + id: 1b44c745-8efe-4aa3-9a78-b4bde3d0e20b + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-target - reference: GPT-3 paper - 06e2c6b1-b6d0-4a05-bcec-004a2bfcf2f2: !Template + - a_good_translation-cs-en-source+target + reference: '' + 2d15416d-0be5-4531-9506-cdbbb1fe8e5b: !Template answer_choices: null - id: 06e2c6b1-b6d0-4a05-bcec-004a2bfcf2f2 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Czech translation is: + id: 2d15416d-0be5-4531-9506-cdbbb1fe8e5b + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-source+target - reference: '' - 16492d65-0760-4b86-86b4-2578994dc9d1: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + 2e3bfc96-c1e1-4180-8cc4-9dfae95d00b6: !Template answer_choices: null - id: 16492d65-0760-4b86-86b4-2578994dc9d1 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: - - ||| {{translation["cs"]}}' + id: 2e3bfc96-c1e1-4180-8cc4-9dfae95d00b6 + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - a_good_translation-en-cs-target reference: '' - 17937d5b-1d1e-4856-9f02-1c33d01e7613: !Template + 30898b95-7fcc-44fe-b41d-2fccd9247d52: !Template answer_choices: null - id: 17937d5b-1d1e-4856-9f02-1c33d01e7613 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 30898b95-7fcc-44fe-b41d-2fccd9247d52 + jinja: '{{translation["cs"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target - reference: '' - 1f5e3b84-8516-4c07-a9a0-52842a4a326b: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3d8622c7-25b5-4a7d-a156-5f6ac289bc64: !Template answer_choices: null - id: 1f5e3b84-8516-4c07-a9a0-52842a4a326b - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: 3d8622c7-25b5-4a7d-a156-5f6ac289bc64 + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target - reference: '' - 35fde963-ed17-4cf4-99e9-050a7c501cd2: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4a609be1-08a1-41f8-a2b4-d166107e1025: !Template answer_choices: null - id: 35fde963-ed17-4cf4-99e9-050a7c501cd2 - jinja: '{{translation["cs"]}} = English: + id: 4a609be1-08a1-41f8-a2b4-d166107e1025 + jinja: 'Translate this into English: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4d33c258-1566-47dd-81c0-1dab0499736d: !Template + - translate-cs-en-target + reference: '' + 50904822-5371-4cd0-9c3c-27bcda65ad93: !Template answer_choices: null - id: 4d33c258-1566-47dd-81c0-1dab0499736d - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: 50904822-5371-4cd0-9c3c-27bcda65ad93 + jinja: 'Translate this from English into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target + - translate-en-cs-source+starget reference: '' - 5926a418-9730-4f3f-a913-16206f7db095: !Template + 516bdcfa-8f19-4d91-b839-723987de3cc8: !Template answer_choices: null - id: 5926a418-9730-4f3f-a913-16206f7db095 - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 516bdcfa-8f19-4d91-b839-723987de3cc8 + jinja: '{{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - translate_as_cs-en-target reference: '' - 5ae66026-566f-4d69-80eb-98bfe9a748c5: !Template + 53af174b-3bd1-41fe-aff5-c72963a8d7d1: !Template answer_choices: null - id: 5ae66026-566f-4d69-80eb-98bfe9a748c5 - jinja: '{{translation["en"]}} = Czech: + id: 53af174b-3bd1-41fe-aff5-c72963a8d7d1 + jinja: 'What is the Czech translation of: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 5c5fc3e1-7125-4305-bbcb-1a28f18eb7d9: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 582fa43b-c399-4e47-bf65-3f6eaac4c2a6: !Template answer_choices: null - id: 5c5fc3e1-7125-4305-bbcb-1a28f18eb7d9 - jinja: 'Czech: {{translation["cs"]}} = English: + id: 582fa43b-c399-4e47-bf65-3f6eaac4c2a6 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -170,12 +166,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 78702895-3efa-43db-9ecd-866d0b2146e9: !Template + - a_good_translation-cs-en-target + reference: '' + 73618299-da46-46a8-992c-4d8a9d178aca: !Template answer_choices: null - id: 78702895-3efa-43db-9ecd-866d0b2146e9 - jinja: 'English: {{translation["en"]}} = Czech: + id: 73618299-da46-46a8-992c-4d8a9d178aca + jinja: '{{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -184,27 +180,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 88a17d2a-d688-4d0e-8b28-a8b10e5ead48: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 843b5407-062b-4135-9930-8bfff41f0500: !Template answer_choices: null - id: 88a17d2a-d688-4d0e-8b28-a8b10e5ead48 - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: 843b5407-062b-4135-9930-8bfff41f0500 + jinja: 'English: {{translation["en"]}} = Czech: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target - reference: '' - 8e4a3e22-78a9-4fa7-94b4-1f9e9cf76467: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 914d4d86-9465-4a70-b5ba-6907c61811f6: !Template answer_choices: null - id: 8e4a3e22-78a9-4fa7-94b4-1f9e9cf76467 - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 914d4d86-9465-4a70-b5ba-6907c61811f6 + jinja: ' {{translation["en"]}} translates into Czech as: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -213,25 +208,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - translate_as_en-cs-target reference: '' - 9003b86f-7c59-459b-a5da-40f1e2ff5782: !Template + a37d598e-8b5c-4cd4-8579-acd131cdf842: !Template answer_choices: null - id: 9003b86f-7c59-459b-a5da-40f1e2ff5782 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: a37d598e-8b5c-4cd4-8579-acd131cdf842 + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target + - version-cs-en-target reference: '' - a0367ec8-1ff9-4d13-9d31-ce212285b2cc: !Template + a7f45664-c9a5-436d-832e-1462094d6402: !Template answer_choices: null - id: a0367ec8-1ff9-4d13-9d31-ce212285b2cc - jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + id: a7f45664-c9a5-436d-832e-1462094d6402 + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version should say: ||| {{translation["cs"]}}' @@ -241,96 +238,99 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-en-cs-target + - version-en-cs-source+target reference: '' - a625334b-415b-4fbb-9b93-2fd523d8d04a: !Template + a9af3ff9-fadb-49d8-95d0-be53093b27e1: !Template answer_choices: null - id: a625334b-415b-4fbb-9b93-2fd523d8d04a - jinja: ' {{translation["en"]}} translates into Czech as: + id: a9af3ff9-fadb-49d8-95d0-be53093b27e1 + jinja: 'Translate this into Czech: {{translation["en"]}} - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target + - translate-en-cs-target reference: '' - ab8f812b-5897-4062-bcfa-531d33948297: !Template + aa7740dd-b61f-4146-a020-8c5bbbf0d287: !Template answer_choices: null - id: ab8f812b-5897-4062-bcfa-531d33948297 - jinja: 'Translate English into Czech: {{translation["en"]}} + id: aa7740dd-b61f-4146-a020-8c5bbbf0d287 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: - ||| {{translation["cs"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-source+starget + - a_good_translation-en-cs-source+target reference: '' - b56a417b-b38a-4ee8-b875-905985ca48b1: !Template + ccf5c2ab-7a25-4b96-96c9-040fcb6ec9db: !Template answer_choices: null - id: b56a417b-b38a-4ee8-b875-905985ca48b1 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: ccf5c2ab-7a25-4b96-96c9-040fcb6ec9db + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - bb32fabf-50aa-4560-be6d-fd31947b9248: !Template + - translate_as_en-cs-source+target + reference: '' + d54087bc-3a16-4a8e-ad86-e1fe067d6b6d: !Template answer_choices: null - id: bb32fabf-50aa-4560-be6d-fd31947b9248 - jinja: 'What is the English translation of : {{translation["cs"]}} + id: d54087bc-3a16-4a8e-ad86-e1fe067d6b6d + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-target - reference: GPT-3 paper - beaf04b3-d4de-4ccf-a13d-f624ac1beaa1: !Template + - how_to_say-en-cs-target + reference: '' + d91a9f47-a8cd-4731-9a3d-d6d5b547014d: !Template answer_choices: null - id: beaf04b3-d4de-4ccf-a13d-f624ac1beaa1 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: d91a9f47-a8cd-4731-9a3d-d6d5b547014d + jinja: 'If the original version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - version-en-cs-target reference: '' - d7376547-b699-4bc7-a753-7896092609eb: !Template + e6200253-6ce1-4179-a8a0-ed7fd9714e62: !Template answer_choices: null - id: d7376547-b699-4bc7-a753-7896092609eb - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: e6200253-6ce1-4179-a8a0-ed7fd9714e62 + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-source+target - reference: GPT 3 paper - f3b839ed-9535-4714-a10a-12436be1f517: !Template + - version-cs-en-source+target + reference: '' + ec893b3e-435c-4557-9c7f-c3c8c2e42a64: !Template answer_choices: null - id: f3b839ed-9535-4714-a10a-12436be1f517 - jinja: '{{translation["cs"]}} translates into English as: + id: ec893b3e-435c-4557-9c7f-c3c8c2e42a64 + jinja: 'How do you say {{translation["cs"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -339,12 +339,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - how_to_say-cs-en-target reference: '' - f764a6f3-bddd-4f63-9a63-faf18aab9c3e: !Template + f2ac2dab-2f82-4b42-a03d-0a4ff8a90549: !Template answer_choices: null - id: f764a6f3-bddd-4f63-9a63-faf18aab9c3e - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: f2ac2dab-2f82-4b42-a03d-0a4ff8a90549 + jinja: 'What is the English translation of : {{translation["cs"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -353,19 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-source+target + - gpt-3-cs-en-target reference: GPT-3 paper - ff8c37a8-4b9f-4a12-8e91-46354e5720fb: !Template + fd1edae5-aa00-4e4c-8941-6ade010da2f0: !Template answer_choices: null - id: ff8c37a8-4b9f-4a12-8e91-46354e5720fb - jinja: 'How do you say {{translation["cs"]}} in English? + id: fd1edae5-aa00-4e4c-8941-6ade010da2f0 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target - reference: '' + - gpt-3-en-cs-source+target + reference: GPT 3 paper diff --git a/promptsource/templates/wmt17/de-en/templates.yaml b/promptsource/templates/wmt17/de-en/templates.yaml index 4b9261304..454089bf2 100644 --- a/promptsource/templates/wmt17/de-en/templates.yaml +++ b/promptsource/templates/wmt17/de-en/templates.yaml @@ -1,95 +1,94 @@ dataset: wmt17 subset: de-en templates: - 08524af5-6d00-4289-a1bb-935aa9af8666: !Template + 00128dd7-093e-4a4a-a08b-f6bf64593060: !Template answer_choices: null - id: 08524af5-6d00-4289-a1bb-935aa9af8666 - jinja: 'What is the English translation of : {{translation["de"]}} + id: 00128dd7-093e-4a4a-a08b-f6bf64593060 + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: GPT-3 paper - 22f21bc1-547f-46c5-8676-b8429d2b242f: !Template + - translate_as_en-de-source+target + reference: '' + 0c9f2ead-1805-4af4-8f68-641946ead546: !Template answer_choices: null - id: 22f21bc1-547f-46c5-8676-b8429d2b242f - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 0c9f2ead-1805-4af4-8f68-641946ead546 + jinja: ' {{translation["en"]}} translates into German as: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate_as_en-de-target reference: '' - 279e6f95-4376-44f2-a4d4-a88320900c9f: !Template + 0de0d489-36bb-4d6e-8e3f-57973cee1461: !Template answer_choices: null - id: 279e6f95-4376-44f2-a4d4-a88320900c9f - jinja: 'How do you say {{translation["en"]}} in German? + id: 0de0d489-36bb-4d6e-8e3f-57973cee1461 + jinja: 'What is the German translation of: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target - reference: '' - 2adfdaef-1e76-4faf-ac39-7a293180f238: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + 199cba3c-a7e7-4205-b681-81dab0fd81c2: !Template answer_choices: null - id: 2adfdaef-1e76-4faf-ac39-7a293180f238 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: 199cba3c-a7e7-4205-b681-81dab0fd81c2 + jinja: 'Translate this from English into German: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target + - translate-en-de-source+starget reference: '' - 33927585-3b79-4575-a14e-cc473d46f552: !Template + 20a1dc77-2e87-4a96-a515-babf03a165e6: !Template answer_choices: null - id: 33927585-3b79-4575-a14e-cc473d46f552 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 20a1dc77-2e87-4a96-a515-babf03a165e6 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - version-en-de-target reference: '' - 465ea4d3-1368-4eaa-9a22-89ec5416ab49: !Template + 37b0220b-f12d-48ba-bdc1-af0493b011fb: !Template answer_choices: null - id: 465ea4d3-1368-4eaa-9a22-89ec5416ab49 - jinja: 'German: {{translation["de"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 37b0220b-f12d-48ba-bdc1-af0493b011fb + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - a_good_translation-en-de-target reference: '' - 4f552a67-50f2-4ec1-8080-2d1fe90d27aa: !Template + 3a23ebf6-31b6-4cbb-a0a8-7872560fd8da: !Template answer_choices: null - id: 4f552a67-50f2-4ec1-8080-2d1fe90d27aa - jinja: 'What is the German translation of: {{translation["en"]}} + id: 3a23ebf6-31b6-4cbb-a0a8-7872560fd8da + jinja: 'Translate this into German: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -98,26 +97,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: GPT-3 paper - 5789ef83-ee21-4277-9bc7-3c95fb8d2ee2: !Template + - translate-en-de-target + reference: '' + 44b0f1c2-c91f-4639-9d4a-2991e7001501: !Template answer_choices: null - id: 5789ef83-ee21-4277-9bc7-3c95fb8d2ee2 - jinja: 'Translate English into German: {{translation["en"]}} + id: 44b0f1c2-c91f-4639-9d4a-2991e7001501 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget + - a_good_translation-de-en-target reference: '' - 5e0b96c1-d9e7-4fee-b905-2977399c70f1: !Template + 4513e0e4-7122-4cb1-8d35-8893cbd276d6: !Template answer_choices: null - id: 5e0b96c1-d9e7-4fee-b905-2977399c70f1 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 4513e0e4-7122-4cb1-8d35-8893cbd276d6 + jinja: 'Translate this into English: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -126,27 +126,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target - reference: GPT-3 paper - 6322d202-a8cb-4ffa-a425-99ae0e55a6ff: !Template + - translate-de-en-target + reference: '' + 6457e3da-34e6-4a9e-b1d5-29006e67478a: !Template answer_choices: null - id: 6322d202-a8cb-4ffa-a425-99ae0e55a6ff - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: 6457e3da-34e6-4a9e-b1d5-29006e67478a + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target - reference: '' - 730615e3-353b-4388-b89f-9c99809b74cd: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + 65b06f84-3937-4d5f-9b07-a97135391b07: !Template answer_choices: null - id: 730615e3-353b-4388-b89f-9c99809b74cd - jinja: '{{translation["en"]}} = German: + id: 65b06f84-3937-4d5f-9b07-a97135391b07 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -155,26 +155,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 76a8f488-3ccd-4455-8083-cf985a4f1c5c: !Template + - a_good_translation-en-de-source+target + reference: '' + 6e84bea7-ed10-43bc-a80b-fbfeba40fec0: !Template answer_choices: null - id: 76a8f488-3ccd-4455-8083-cf985a4f1c5c - jinja: '{{translation["de"]}} = English: + id: 6e84bea7-ed10-43bc-a80b-fbfeba40fec0 + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 78ac7c05-174a-4bb6-b41a-bff529679779: !Template + - version-en-de-source+target + reference: '' + 6f76439a-5335-4880-8072-0c02671c3b68: !Template answer_choices: null - id: 78ac7c05-174a-4bb6-b41a-bff529679779 - jinja: ' {{translation["en"]}} translates into German as: + id: 6f76439a-5335-4880-8072-0c02671c3b68 + jinja: '{{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -183,39 +184,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target - reference: '' - 889e5533-28c8-4094-9127-103d4741a341: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 79613dca-7808-4ada-9f2b-5502f4201001: !Template answer_choices: null - id: 889e5533-28c8-4094-9127-103d4741a341 - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 79613dca-7808-4ada-9f2b-5502f4201001 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target - reference: '' - 8ef91b8e-9cda-4828-b26b-a474c03004b9: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 86d73059-130f-42e4-a765-e2d53ef72de1: !Template answer_choices: null - id: 8ef91b8e-9cda-4828-b26b-a474c03004b9 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 86d73059-130f-42e4-a765-e2d53ef72de1 + jinja: 'Translate this from German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target - reference: '' - a4448a0c-f546-4332-83b6-fd78e415c0be: !Template + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8882e270-2797-4d00-9061-3da89ebb0c15: !Template answer_choices: null - id: a4448a0c-f546-4332-83b6-fd78e415c0be + id: 8882e270-2797-4d00-9061-3da89ebb0c15 jinja: '{{translation["de"]}} translates into English as: ||| {{translation["en"]}}' @@ -227,54 +228,51 @@ templates: name: !!python/tuple - translate_as_de-en-target reference: '' - b3d26334-534b-4d2f-b244-eb6486081457: !Template + 8be79e83-710b-4669-b769-e5669c63831f: !Template answer_choices: null - id: b3d26334-534b-4d2f-b244-eb6486081457 - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 8be79e83-710b-4669-b769-e5669c63831f + jinja: 'What is the English translation of : {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-source+target - reference: GPT 3 paper - b8bf6d86-a3be-4d00-ac77-e522b98b60c1: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + 97584294-7bb0-4434-aa64-eea905210e2a: !Template answer_choices: null - id: b8bf6d86-a3be-4d00-ac77-e522b98b60c1 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: - - ||| {{translation["en"]}}' + id: 97584294-7bb0-4434-aa64-eea905210e2a + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - a_good_translation-de-en-source+target reference: '' - c0c0eb2c-1fd1-42fc-b8de-7c2885d3eefb: !Template + a46ff7ba-7890-4da2-85fd-ca8b8fc54dda: !Template answer_choices: null - id: c0c0eb2c-1fd1-42fc-b8de-7c2885d3eefb - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: + id: a46ff7ba-7890-4da2-85fd-ca8b8fc54dda + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - how_to_say-de-en-target reference: '' - c9f3b71f-77eb-4cb6-897a-257b8ac842da: !Template + afb88539-be1a-40e7-8b1e-8b4d9b035cc3: !Template answer_choices: null - id: c9f3b71f-77eb-4cb6-897a-257b8ac842da - jinja: 'How do you say {{translation["de"]}} in English? + id: afb88539-be1a-40e7-8b1e-8b4d9b035cc3 + jinja: 'German: {{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -283,89 +281,91 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target - reference: '' - e1a1c055-9dff-4a98-9a90-38338fbc0174: !Template + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d1f4b6ac-e930-43ec-8970-c95adcf4b2af: !Template answer_choices: null - id: e1a1c055-9dff-4a98-9a90-38338fbc0174 - jinja: 'Translate into German: {{translation["en"]}} + id: d1f4b6ac-e930-43ec-8970-c95adcf4b2af + jinja: '{{translation["de"]}} = English: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target - reference: '' - e5003513-1a97-41d4-ada8-15b9543e47b7: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d6f06333-44b3-45fe-9161-8f26af62e8c6: !Template answer_choices: null - id: e5003513-1a97-41d4-ada8-15b9543e47b7 - jinja: 'German: {{translation["de"]}} = English: + id: d6f06333-44b3-45fe-9161-8f26af62e8c6 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - ea701736-3e22-4eb8-9bb1-182b12b54918: !Template + - how_to_say-en-de-target + reference: '' + d72bb3dd-48de-47cb-97a5-bdf03d8246f7: !Template answer_choices: null - id: ea701736-3e22-4eb8-9bb1-182b12b54918 - jinja: 'English: {{translation["en"]}} = German: + id: d72bb3dd-48de-47cb-97a5-bdf03d8246f7 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - efc189f5-ac49-4615-89f4-f343fb7b5e4c: !Template + - version-de-en-target + reference: '' + dc567cdc-14ec-4b86-8282-2e4cfe9cc45b: !Template answer_choices: null - id: efc189f5-ac49-4615-89f4-f343fb7b5e4c - jinja: 'Translate into English: {{translation["de"]}} + id: dc567cdc-14ec-4b86-8282-2e4cfe9cc45b + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target - reference: '' - f773be03-b0c4-439a-874f-00a58568c8b5: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + ea5b141a-0097-4aa4-9ab5-e61e7e691fbe: !Template answer_choices: null - id: f773be03-b0c4-439a-874f-00a58568c8b5 - jinja: 'Translate German into English: {{translation["de"]}} + id: ea5b141a-0097-4aa4-9ab5-e61e7e691fbe + jinja: 'German: {{translation["de"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - ffa3a920-0622-46ec-a234-017add93712c: !Template + - translate_as_de-en-source+target + reference: '' + ebec0874-7b1d-4191-848c-d55d5c6f10fd: !Template answer_choices: null - id: ffa3a920-0622-46ec-a234-017add93712c - jinja: 'English: {{translation["en"]}} translates into German as: + id: ebec0874-7b1d-4191-848c-d55d5c6f10fd + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - version-de-en-source+target reference: '' diff --git a/promptsource/templates/wmt17/fi-en/templates.yaml b/promptsource/templates/wmt17/fi-en/templates.yaml index effc56b1d..d37368c08 100644 --- a/promptsource/templates/wmt17/fi-en/templates.yaml +++ b/promptsource/templates/wmt17/fi-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt17 subset: fi-en templates: - 09d937c7-15dc-4699-9971-4900c643fc3b: !Template + 0721d787-db52-4003-a644-efea878a23f0: !Template answer_choices: null - id: 09d937c7-15dc-4699-9971-4900c643fc3b - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 0721d787-db52-4003-a644-efea878a23f0 + jinja: 'Translate this into English: {{translation["fi"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -13,139 +13,137 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: GPT-3 paper - 1030aa90-aa70-4d1f-a819-a44f278d13db: !Template + - translate-fi-en-target + reference: '' + 0d23b84d-10f9-43f0-9972-c4f1273d0230: !Template answer_choices: null - id: 1030aa90-aa70-4d1f-a819-a44f278d13db - jinja: '{{translation["fi"]}} = English: + id: 0d23b84d-10f9-43f0-9972-c4f1273d0230 + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 1a7e4164-f420-4845-b209-04d017592cb7: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + 164c33b9-a1c2-457a-b8e6-fb4e03a4ac14: !Template answer_choices: null - id: 1a7e4164-f420-4845-b209-04d017592cb7 - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: 164c33b9-a1c2-457a-b8e6-fb4e03a4ac14 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-source+target - reference: GPT 3 paper - 1b06cda4-384c-4e4f-848d-d2e4ecfbc0eb: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 1ab09eb2-48c9-4f48-96ea-d3bf8875a363: !Template answer_choices: null - id: 1b06cda4-384c-4e4f-848d-d2e4ecfbc0eb - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: 1ab09eb2-48c9-4f48-96ea-d3bf8875a363 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target - reference: '' - 21f84cd6-a91c-42e6-b8ad-9d184b95bae1: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 1d028db9-5f27-4ff6-a9bf-3b8f7e8f49eb: !Template answer_choices: null - id: 21f84cd6-a91c-42e6-b8ad-9d184b95bae1 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 1d028db9-5f27-4ff6-a9bf-3b8f7e8f49eb + jinja: 'Translate this into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - translate-en-fi-target reference: '' - 2606cfdf-7f82-4664-891a-326fc7fc0e6f: !Template + 1fea6184-a880-4f5a-9856-ad825bb2ad86: !Template answer_choices: null - id: 2606cfdf-7f82-4664-891a-326fc7fc0e6f - jinja: 'How do you say {{translation["fi"]}} in English? + id: 1fea6184-a880-4f5a-9856-ad825bb2ad86 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target - reference: '' - 2edd1c4f-9555-4b8f-8e0b-a2bab6be396a: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 22a056a5-9ac1-4f87-83df-3ae8326b4693: !Template answer_choices: null - id: 2edd1c4f-9555-4b8f-8e0b-a2bab6be396a - jinja: ' {{translation["en"]}} translates into Finnish as: - - ||| {{translation["fi"]}}' + id: 22a056a5-9ac1-4f87-83df-3ae8326b4693 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - a_good_translation-en-fi-target reference: '' - 3a1ccb5c-d1d4-49e4-a9d5-9e048b64dc66: !Template + 281fb66a-d0f2-40a7-a6e8-4c5844bed10c: !Template answer_choices: null - id: 3a1ccb5c-d1d4-49e4-a9d5-9e048b64dc66 - jinja: 'Finnish: {{translation["fi"]}} = English: + id: 281fb66a-d0f2-40a7-a6e8-4c5844bed10c + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 424c292d-22f8-4dbe-9f81-6c5dc6bb788c: !Template + - translate_as_en-fi-target + reference: '' + 3b679f65-aa5b-4bc2-bb47-b5dada7fb0f6: !Template answer_choices: null - id: 424c292d-22f8-4dbe-9f81-6c5dc6bb788c - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 3b679f65-aa5b-4bc2-bb47-b5dada7fb0f6 + jinja: 'How do you say {{translation["en"]}} in Finnish? - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 4ea3f2d1-3fd3-44fe-87cb-c2969b783433: !Template + - how_to_say-en-fi-target + reference: '' + 3d1b6cef-c40e-4965-ad0b-2795b15a3f45: !Template answer_choices: null - id: 4ea3f2d1-3fd3-44fe-87cb-c2969b783433 - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: 3d1b6cef-c40e-4965-ad0b-2795b15a3f45 + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target + - gpt-3-fi-en-target reference: GPT-3 paper - 62b47c3d-8021-4160-8874-3eddb54cd621: !Template + 50d7b5e4-8781-4f74-b710-ac0e7b14a763: !Template answer_choices: null - id: 62b47c3d-8021-4160-8874-3eddb54cd621 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: 50d7b5e4-8781-4f74-b710-ac0e7b14a763 + jinja: 'What is the Finnish translation of: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -154,12 +152,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target - reference: '' - 6a54d46e-2823-4946-b387-2281a0de107a: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 5c9db42a-4000-422d-ba74-8cec1f12cf3e: !Template answer_choices: null - id: 6a54d46e-2823-4946-b387-2281a0de107a - jinja: '{{translation["fi"]}} translates into English as: + id: 5c9db42a-4000-422d-ba74-8cec1f12cf3e + jinja: 'Finnish: {{translation["fi"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -168,80 +166,81 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target + - translate_as_fi-en-source+target reference: '' - 70707354-7e23-4c53-99c3-d2b7151ac6c5: !Template + 5d973ade-560b-4813-9c76-a5a1e415da85: !Template answer_choices: null - id: 70707354-7e23-4c53-99c3-d2b7151ac6c5 - jinja: 'Translate into English: {{translation["fi"]}} + id: 5d973ade-560b-4813-9c76-a5a1e415da85 + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - translate_as_en-fi-source+target reference: '' - 8b8ee16f-42e9-4253-b128-269d9191b130: !Template + 67bc9eb3-0769-4357-8b2f-15b8c3b9abf7: !Template answer_choices: null - id: 8b8ee16f-42e9-4253-b128-269d9191b130 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 67bc9eb3-0769-4357-8b2f-15b8c3b9abf7 + jinja: 'Translate this from English into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target + - translate-en-fi-source+starget reference: '' - 8ca1100e-4741-4bad-bf26-c5af06f35eb7: !Template + 8053805e-fc9d-43e8-82d4-e30e3a76509a: !Template answer_choices: null - id: 8ca1100e-4741-4bad-bf26-c5af06f35eb7 - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 8053805e-fc9d-43e8-82d4-e30e3a76509a + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-source+starget + - how_to_say-fi-en-target reference: '' - b71c4561-eea1-4b01-a4a3-d795e32796c5: !Template + 882f5ee3-d108-4811-9876-0972b437e45e: !Template answer_choices: null - id: b71c4561-eea1-4b01-a4a3-d795e32796c5 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 882f5ee3-d108-4811-9876-0972b437e45e + jinja: 'Translate this from Finnish into English: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target - reference: '' - c0f12c08-16cf-4f04-bf21-aea1b895fbf4: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 92ecb048-a030-427f-96c7-86732efe4245: !Template answer_choices: null - id: c0f12c08-16cf-4f04-bf21-aea1b895fbf4 - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 92ecb048-a030-427f-96c7-86732efe4245 + jinja: 'English: {{translation["en"]}} = Finnish: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target - reference: '' - c42e3493-bfaa-4143-9a78-ee4a09785bcd: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + a788c720-369e-439f-8805-4a18530044d7: !Template answer_choices: null - id: c42e3493-bfaa-4143-9a78-ee4a09785bcd + id: a788c720-369e-439f-8805-4a18530044d7 jinja: 'If the original version says: {{translation["en"]}}; then the Finnish version should say: @@ -254,67 +253,68 @@ templates: name: !!python/tuple - version-en-fi-target reference: '' - c5ab6e25-8318-46b3-8505-db0bb87ddf0a: !Template + b12a7f30-947c-4ee6-80ae-ff439beaab15: !Template answer_choices: null - id: c5ab6e25-8318-46b3-8505-db0bb87ddf0a - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: b12a7f30-947c-4ee6-80ae-ff439beaab15 + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - a_good_translation-fi-en-source+target reference: '' - c5dde640-4a2c-46ca-bff1-20ac8e506697: !Template + b3dd87d1-9c83-4350-b0e2-bf1a78222aa7: !Template answer_choices: null - id: c5dde640-4a2c-46ca-bff1-20ac8e506697 - jinja: '{{translation["en"]}} = Finnish: + id: b3dd87d1-9c83-4350-b0e2-bf1a78222aa7 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-target + - xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - dcb7eee2-776a-43fd-b41e-047875c89b32: !Template + ceddd576-ee41-4e4d-8b9d-e1ecb9bfcc68: !Template answer_choices: null - id: dcb7eee2-776a-43fd-b41e-047875c89b32 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: ceddd576-ee41-4e4d-8b9d-e1ecb9bfcc68 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target + - version-fi-en-target reference: '' - ddc72d31-5ec6-4d4f-b844-14014b4214eb: !Template + d031d8ac-0fa0-40ba-bb61-b9da287bb8cf: !Template answer_choices: null - id: ddc72d31-5ec6-4d4f-b844-14014b4214eb - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: d031d8ac-0fa0-40ba-bb61-b9da287bb8cf + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-source+target - reference: GPT-3 paper - e6e2abd5-6778-46b4-8406-a3c267999ff9: !Template + - version-en-fi-source+target + reference: '' + d2e78da8-9a4b-499a-b7fa-5802fd1a5fdd: !Template answer_choices: null - id: e6e2abd5-6778-46b4-8406-a3c267999ff9 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: d2e78da8-9a4b-499a-b7fa-5802fd1a5fdd + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -323,49 +323,49 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - a_good_translation-en-fi-source+target reference: '' - eae9555c-c7a8-45ed-9061-efb510c06d84: !Template + d42a1dda-15e1-4630-be21-4de22e03dcec: !Template answer_choices: null - id: eae9555c-c7a8-45ed-9061-efb510c06d84 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: d42a1dda-15e1-4630-be21-4de22e03dcec + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target + - a_good_translation-fi-en-target reference: '' - f0af9fb5-b096-41af-bd4c-6517c1e20c66: !Template + e8db7580-1543-4088-9cdb-f75879490e6c: !Template answer_choices: null - id: f0af9fb5-b096-41af-bd4c-6517c1e20c66 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: e8db7580-1543-4088-9cdb-f75879490e6c + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target + - version-fi-en-source+target reference: '' - fefbc6f1-438a-43d5-aad7-07f7cdb96c55: !Template + f8a9585e-dd83-4941-ada1-2125b81637d6: !Template answer_choices: null - id: fefbc6f1-438a-43d5-aad7-07f7cdb96c55 - jinja: 'English: {{translation["en"]}} = Finnish: + id: f8a9585e-dd83-4941-ada1-2125b81637d6 + jinja: '{{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + - translate_as_fi-en-target + reference: '' diff --git a/promptsource/templates/wmt17/lv-en/templates.yaml b/promptsource/templates/wmt17/lv-en/templates.yaml index 3ed2015dd..d6a7c6cea 100644 --- a/promptsource/templates/wmt17/lv-en/templates.yaml +++ b/promptsource/templates/wmt17/lv-en/templates.yaml @@ -1,39 +1,39 @@ dataset: wmt17 subset: lv-en templates: - 08b25640-bb2b-4521-bd62-cd97e4246ed3: !Template + 017128c4-7010-4477-a16d-b236766d7944: !Template answer_choices: null - id: 08b25640-bb2b-4521-bd62-cd97e4246ed3 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Latvian translation is: + id: 017128c4-7010-4477-a16d-b236766d7944 + jinja: 'Given the following passage: {{translation["lv"]}} , a good English translation + is: - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lv-source+target + - a_good_translation-lv-en-target reference: '' - 10e91e3c-0b78-4c82-8d1b-44678b8aed83: !Template + 0936fbe5-6138-437d-9ccc-44d02545d02e: !Template answer_choices: null - id: 10e91e3c-0b78-4c82-8d1b-44678b8aed83 - jinja: 'Translate English into Latvian: {{translation["en"]}} + id: 0936fbe5-6138-437d-9ccc-44d02545d02e + jinja: 'Translate this into English: {{translation["lv"]}} - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lv-source+starget + - translate-lv-en-target reference: '' - 19b5b477-e5db-4a2c-a11b-afbb59e69f25: !Template + 09e7d1ab-2efb-4130-a68c-252c483e6d44: !Template answer_choices: null - id: 19b5b477-e5db-4a2c-a11b-afbb59e69f25 - jinja: 'Latvian: {{translation["lv"]}} translates into English as: + id: 09e7d1ab-2efb-4130-a68c-252c483e6d44 + jinja: '{{translation["lv"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -42,156 +42,153 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_lv-en-source+target - reference: '' - 1c81d65c-1b56-4e2d-a8f2-c230c1e9f0d5: !Template + - xglm-lv-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 1b20a572-10df-47ad-8318-b7d367de2a06: !Template answer_choices: null - id: 1c81d65c-1b56-4e2d-a8f2-c230c1e9f0d5 - jinja: 'If the English version says: {{translation["en"]}}; then the Latvian version - should say: + id: 1b20a572-10df-47ad-8318-b7d367de2a06 + jinja: 'What is the Latvian translation of: {{translation["en"]}} - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lv-source+target - reference: '' - 1d87c64f-f016-4437-95cd-f385869a121b: !Template + - gpt-3-en-lv-target + reference: GPT-3 paper + 1c127357-54d2-434e-a346-c6116b7ae131: !Template answer_choices: null - id: 1d87c64f-f016-4437-95cd-f385869a121b - jinja: 'If the original version says: {{translation["en"]}}; then the Latvian - version should say: + id: 1c127357-54d2-434e-a346-c6116b7ae131 + jinja: 'What is the Latvian translation of the English sentence: {{translation["en"]}} - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lv-target - reference: '' - 2b42a4aa-66c5-49e2-93f6-15250e2ab546: !Template + - gpt-3-en-lv-source+target + reference: GPT 3 paper + 1dddcbde-f11c-44bd-a6dc-abc14c9ff8d2: !Template answer_choices: null - id: 2b42a4aa-66c5-49e2-93f6-15250e2ab546 - jinja: 'What is the Latvian translation of the English sentence: {{translation["en"]}} + id: 1dddcbde-f11c-44bd-a6dc-abc14c9ff8d2 + jinja: 'English: {{translation["en"]}} translates into Latvian as: - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lv-source+target - reference: GPT 3 paper - 35e1ebe3-6eb5-4fe2-871c-a722331ea2d5: !Template + - translate_as_en-lv-source+target + reference: '' + 2201a3af-bb28-4cb7-afc0-dfeecef0def2: !Template answer_choices: null - id: 35e1ebe3-6eb5-4fe2-871c-a722331ea2d5 - jinja: 'If the original version says: {{translation["lv"]}}; then the English - version should say: + id: 2201a3af-bb28-4cb7-afc0-dfeecef0def2 + jinja: 'What is the English translation of : {{translation["lv"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lv-en-target - reference: '' - 4564679c-9033-4791-8bde-5b215ba17f63: !Template + - gpt-3-lv-en-target + reference: GPT-3 paper + 25e58c80-4a60-4477-b406-cf7d9cc6dd57: !Template answer_choices: null - id: 4564679c-9033-4791-8bde-5b215ba17f63 - jinja: 'Given the following passage: {{translation["lv"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: 25e58c80-4a60-4477-b406-cf7d9cc6dd57 + jinja: 'Given the following source text in Latvian: {{translation["lv"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lv-en-target + - a_good_translation-lv-en-source+target reference: '' - 4a16394d-aee6-4df5-8704-d8854294e79f: !Template + 2b168e9b-b592-4ef8-aff0-74a06059526f: !Template answer_choices: null - id: 4a16394d-aee6-4df5-8704-d8854294e79f - jinja: '{{translation["en"]}} = Latvian: + id: 2b168e9b-b592-4ef8-aff0-74a06059526f + jinja: '{{translation["lv"]}} translates into English as: - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-lv-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4b9dceb5-49dd-4354-a1b2-57059bb0b1b2: !Template + - translate_as_lv-en-target + reference: '' + 3a396647-7b0e-4d4c-8e9c-454343a532af: !Template answer_choices: null - id: 4b9dceb5-49dd-4354-a1b2-57059bb0b1b2 - jinja: ' {{translation["en"]}} translates into Latvian as: + id: 3a396647-7b0e-4d4c-8e9c-454343a532af + jinja: 'Latvian: {{translation["lv"]}} = English: - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lv-target - reference: '' - 61290165-dec6-437b-80c4-ec4350b11102: !Template + - xglm-lv-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 442985e4-d28c-4814-aa88-51582ea4bd8a: !Template answer_choices: null - id: 61290165-dec6-437b-80c4-ec4350b11102 - jinja: 'Translate into English: {{translation["lv"]}} + id: 442985e4-d28c-4814-aa88-51582ea4bd8a + jinja: 'How do you say {{translation["lv"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lv-en-target + - how_to_say-lv-en-target reference: '' - 664cf31e-17a3-4ad5-92f5-5d38063b25f6: !Template + 4630798f-5bff-412d-bac3-1eb26977cb43: !Template answer_choices: null - id: 664cf31e-17a3-4ad5-92f5-5d38063b25f6 - jinja: 'What is the English translation of : {{translation["lv"]}} + id: 4630798f-5bff-412d-bac3-1eb26977cb43 + jinja: 'If the Latvian version says: {{translation["lv"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lv-en-target - reference: GPT-3 paper - 78a53869-2df4-45c0-9506-a640eaf2073c: !Template + - version-lv-en-source+target + reference: '' + 4a89beac-0b24-4929-a60b-86e67bdf89d1: !Template answer_choices: null - id: 78a53869-2df4-45c0-9506-a640eaf2073c - jinja: '{{translation["lv"]}} translates into English as: + id: 4a89beac-0b24-4929-a60b-86e67bdf89d1 + jinja: 'If the original version says: {{translation["en"]}}; then the Latvian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_lv-en-target + - version-en-lv-target reference: '' - 810d05bf-6851-42d4-a62a-80a3a2b34066: !Template + 5159f3c9-4cef-4e23-b0c6-101d27d1a561: !Template answer_choices: null - id: 810d05bf-6851-42d4-a62a-80a3a2b34066 - jinja: '{{translation["lv"]}} = English: + id: 5159f3c9-4cef-4e23-b0c6-101d27d1a561 + jinja: 'Latvian: {{translation["lv"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -200,55 +197,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-lv-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 84586c6e-9f21-4219-aaef-acd3e580964c: !Template + - translate_as_lv-en-source+target + reference: '' + 5358c036-b6b0-48ad-a8fb-8296c35a53ea: !Template answer_choices: null - id: 84586c6e-9f21-4219-aaef-acd3e580964c - jinja: 'Translate into Latvian: {{translation["en"]}} + id: 5358c036-b6b0-48ad-a8fb-8296c35a53ea + jinja: 'What is the English translation of the Latvian sentence: {{translation["lv"]}} - ||| {{translation["lv"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lv-target - reference: '' - 894c74f2-7e1c-41c7-9fd8-f21c0931ce38: !Template + - gpt-3-lv-en-source+target + reference: GPT-3 paper + 5c6c9aec-1736-448a-9036-f0f5436838cc: !Template answer_choices: null - id: 894c74f2-7e1c-41c7-9fd8-f21c0931ce38 - jinja: 'If the Latvian version says: {{translation["lv"]}}; then the English version - should say: + id: 5c6c9aec-1736-448a-9036-f0f5436838cc + jinja: 'Translate this from English into Latvian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lv-en-source+target + - translate-en-lv-source+starget reference: '' - 9536897d-9ebc-4340-bba3-7956d9ac60a1: !Template + 79f2149d-78fe-43f0-bb7e-aadc75d5b045: !Template answer_choices: null - id: 9536897d-9ebc-4340-bba3-7956d9ac60a1 - jinja: 'What is the Latvian translation of: {{translation["en"]}} + id: 79f2149d-78fe-43f0-bb7e-aadc75d5b045 + jinja: '{{translation["en"]}} = Latvian: - ||| {{translation["lv"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lv-target - reference: GPT-3 paper - 9dae3dbd-3828-4589-8a0f-c69af4b724b3: !Template + - xglm-en-lv-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 854685ec-5e2e-46f7-b47e-a3a5502179b5: !Template answer_choices: null - id: 9dae3dbd-3828-4589-8a0f-c69af4b724b3 - jinja: 'English: {{translation["en"]}} = Latvian: + id: 854685ec-5e2e-46f7-b47e-a3a5502179b5 + jinja: ' {{translation["en"]}} translates into Latvian as: ||| {{translation["lv"]}}' metadata: !TemplateMetadata @@ -257,40 +253,42 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-lv-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - a334cdf7-eedf-4222-bc93-cde54276f636: !Template + - translate_as_en-lv-target + reference: '' + 87b88026-d3c8-4b36-b6fc-3754883ce195: !Template answer_choices: null - id: a334cdf7-eedf-4222-bc93-cde54276f636 - jinja: 'Translate Latvian into English: {{translation["lv"]}} + id: 87b88026-d3c8-4b36-b6fc-3754883ce195 + jinja: 'If the English version says: {{translation["en"]}}; then the Latvian version + should say: - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lv-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - bf88be1e-ffdb-4544-a205-69b9d7b4264d: !Template + - version-en-lv-source+target + reference: '' + 912ea72b-7973-44bc-afb8-4635fd55a838: !Template answer_choices: null - id: bf88be1e-ffdb-4544-a205-69b9d7b4264d - jinja: 'What is the English translation of the Latvian sentence: {{translation["lv"]}} + id: 912ea72b-7973-44bc-afb8-4635fd55a838 + jinja: 'How do you say {{translation["en"]}} in Latvian? - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lv-en-source+target - reference: GPT-3 paper - c3da2972-d194-4fd6-8080-31f10466e9ab: !Template + - how_to_say-en-lv-target + reference: '' + a7605476-3722-4b9c-80c3-f1fa063d63f5: !Template answer_choices: null - id: c3da2972-d194-4fd6-8080-31f10466e9ab - jinja: 'How do you say {{translation["en"]}} in Latvian? + id: a7605476-3722-4b9c-80c3-f1fa063d63f5 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Latvian translation is: ||| {{translation["lv"]}}' metadata: !TemplateMetadata @@ -299,73 +297,75 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-lv-target + - a_good_translation-en-lv-source+target reference: '' - c8fc6565-7b8c-45b8-afd9-a48ca71cd6d8: !Template + c7733bca-74bd-478b-b4a0-ae399b8ee720: !Template answer_choices: null - id: c8fc6565-7b8c-45b8-afd9-a48ca71cd6d8 - jinja: 'Given the following source text in Latvian: {{translation["lv"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: c7733bca-74bd-478b-b4a0-ae399b8ee720 + jinja: 'Translate this from Latvian into English: {{translation["lv"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lv-en-source+target - reference: '' - ce70bac8-4e16-482b-b211-974f455a86b7: !Template + - translate-lv-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + d3d7c0cf-57bf-4067-9229-529ea7a520f5: !Template answer_choices: null - id: ce70bac8-4e16-482b-b211-974f455a86b7 - jinja: 'Latvian: {{translation["lv"]}} = English: + id: d3d7c0cf-57bf-4067-9229-529ea7a520f5 + jinja: 'English: {{translation["en"]}} = Latvian: - ||| {{translation["en"]}}' + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-lv-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d40be9ac-abae-4d6b-ba60-1fdac62fe5ba: !Template + - xglm-en-lv-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + f2924798-0423-43f9-b203-251dacec9a7a: !Template answer_choices: null - id: d40be9ac-abae-4d6b-ba60-1fdac62fe5ba - jinja: 'Given the following passage: {{translation["en"]}} , a good Latvian translation - is: ||| {{translation["lv"]}}' + id: f2924798-0423-43f9-b203-251dacec9a7a + jinja: 'Translate this into Latvian: {{translation["en"]}} + + ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lv-target + - translate-en-lv-target reference: '' - e3e8d9d1-3b6d-4739-97aa-234bd0aa1261: !Template + f32085fc-eb1b-4383-88b7-e9eea44fa379: !Template answer_choices: null - id: e3e8d9d1-3b6d-4739-97aa-234bd0aa1261 - jinja: 'How do you say {{translation["lv"]}} in English? + id: f32085fc-eb1b-4383-88b7-e9eea44fa379 + jinja: 'If the original version says: {{translation["lv"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-lv-en-target + - version-lv-en-target reference: '' - eb064e2e-da31-416e-b93a-47001fce3217: !Template + fa246f5b-3a3d-44a1-9f3e-502572ead9a6: !Template answer_choices: null - id: eb064e2e-da31-416e-b93a-47001fce3217 - jinja: 'English: {{translation["en"]}} translates into Latvian as: - - ||| {{translation["lv"]}}' + id: fa246f5b-3a3d-44a1-9f3e-502572ead9a6 + jinja: 'Given the following passage: {{translation["en"]}} , a good Latvian translation + is: ||| {{translation["lv"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lv-source+target + - a_good_translation-en-lv-target reference: '' diff --git a/promptsource/templates/wmt17/ru-en/templates.yaml b/promptsource/templates/wmt17/ru-en/templates.yaml index 3706fd030..523cd3b1f 100644 --- a/promptsource/templates/wmt17/ru-en/templates.yaml +++ b/promptsource/templates/wmt17/ru-en/templates.yaml @@ -1,108 +1,111 @@ dataset: wmt17 subset: ru-en templates: - 0607296e-2db7-4eaf-861a-3a9a9e804b6e: !Template + 04cd239b-bad3-4feb-8239-3fcb9b6090d0: !Template answer_choices: null - id: 0607296e-2db7-4eaf-861a-3a9a9e804b6e - jinja: '{{translation["ru"]}} translates into English as: + id: 04cd239b-bad3-4feb-8239-3fcb9b6090d0 + jinja: 'English: {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - translate_as_en-ru-source+target reference: '' - 0f39c75a-ea5b-4327-bb94-dbc98bfe53d2: !Template + 0f53aa20-a3ff-4dcc-8178-0ff8563491bd: !Template answer_choices: null - id: 0f39c75a-ea5b-4327-bb94-dbc98bfe53d2 - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 0f53aa20-a3ff-4dcc-8178-0ff8563491bd + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target + - translate_as_ru-en-target reference: '' - 0f6bec96-40ff-4a8d-bb23-8b2e5dc4c45d: !Template + 1cb0dfa5-cac2-4a05-aa0e-48a4796f8a81: !Template answer_choices: null - id: 0f6bec96-40ff-4a8d-bb23-8b2e5dc4c45d - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 1cb0dfa5-cac2-4a05-aa0e-48a4796f8a81 + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - 119a637e-be90-4e0c-a020-bdfeeb2269a3: !Template + - a_good_translation-ru-en-target + reference: '' + 2274b7e4-9d86-4f1c-a945-604deeae3e8c: !Template answer_choices: null - id: 119a637e-be90-4e0c-a020-bdfeeb2269a3 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 2274b7e4-9d86-4f1c-a945-604deeae3e8c + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target + - translate_as_ru-en-source+target reference: '' - 122aab6c-3067-4605-9aa0-ae3c1e38be82: !Template + 2babd6e7-8d10-4eb9-93be-7d27b3e675b9: !Template answer_choices: null - id: 122aab6c-3067-4605-9aa0-ae3c1e38be82 - jinja: 'Russian: {{translation["ru"]}} = English: + id: 2babd6e7-8d10-4eb9-93be-7d27b3e675b9 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 232d0ee6-75f6-40af-a1f4-cfa601ef902f: !Template + - version-en-ru-source+target + reference: '' + 31b8fd81-e2f9-4a1a-827d-efd85356ba09: !Template answer_choices: null - id: 232d0ee6-75f6-40af-a1f4-cfa601ef902f - jinja: 'Translate into Russian: {{translation["en"]}} + id: 31b8fd81-e2f9-4a1a-827d-efd85356ba09 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - version-ru-en-target reference: '' - 2e1aaad4-43a0-4e7d-bcaa-4a1b127fbc9e: !Template + 352474e6-f128-40e6-ba92-2d9b5d259796: !Template answer_choices: null - id: 2e1aaad4-43a0-4e7d-bcaa-4a1b127fbc9e - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 352474e6-f128-40e6-ba92-2d9b5d259796 + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - 33421b68-3dcc-4cba-8dae-fb5748bde1b3: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3fbb2028-f9b2-45dd-bc56-c77fdf27bb67: !Template answer_choices: null - id: 33421b68-3dcc-4cba-8dae-fb5748bde1b3 - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 3fbb2028-f9b2-45dd-bc56-c77fdf27bb67 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -111,11 +114,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-source+starget - reference: '' - 42ec8f17-2b4a-4519-96af-29e00085fb90: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 5dd35a60-5609-40bb-873e-4a3052be1dfc: !Template answer_choices: null - id: 42ec8f17-2b4a-4519-96af-29e00085fb90 + id: 5dd35a60-5609-40bb-873e-4a3052be1dfc jinja: 'Given the following source text in English: {{translation["en"]}} , a good Russian translation is: @@ -128,69 +131,80 @@ templates: name: !!python/tuple - a_good_translation-en-ru-source+target reference: '' - 4471f796-01ca-4b57-b553-e7e5954f3b49: !Template + 66bf33ae-0cf8-4649-8ec8-40fcb4ab2e8f: !Template answer_choices: null - id: 4471f796-01ca-4b57-b553-e7e5954f3b49 - jinja: ' {{translation["en"]}} translates into Russian as: + id: 66bf33ae-0cf8-4649-8ec8-40fcb4ab2e8f + jinja: 'Translate this from English into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate-en-ru-source+starget reference: '' - 4cf94fd2-3ad5-46cd-ace5-5b4b5a20df9b: !Template + 6b05359f-8d2e-467b-ab2f-2bfafd9020a3: !Template answer_choices: null - id: 4cf94fd2-3ad5-46cd-ace5-5b4b5a20df9b - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 6b05359f-8d2e-467b-ab2f-2bfafd9020a3 + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target - reference: '' - 53709362-7f93-4e4b-956a-21c967c34319: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 7489148f-3d36-4bfb-831b-6dc776e09d0a: !Template answer_choices: null - id: 53709362-7f93-4e4b-956a-21c967c34319 - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 7489148f-3d36-4bfb-831b-6dc776e09d0a + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target - reference: '' - 55297bd5-a3b1-4cdb-92d8-f72ffd58545e: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 79e2e60d-2905-4718-80c4-702b880bed86: !Template answer_choices: null - id: 55297bd5-a3b1-4cdb-92d8-f72ffd58545e - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: 79e2e60d-2905-4718-80c4-702b880bed86 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 7a2332c9-4e93-4338-90f7-32dd73a13454: !Template + answer_choices: null + id: 7a2332c9-4e93-4338-90f7-32dd73a13454 + jinja: 'How do you say {{translation["en"]}} in Russian? + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - how_to_say-en-ru-target reference: '' - 5d092fdc-0c1d-4a3f-b619-c62eff356499: !Template + 8552f282-7ec0-43f1-9055-afc034676c3f: !Template answer_choices: null - id: 5d092fdc-0c1d-4a3f-b619-c62eff356499 - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 8552f282-7ec0-43f1-9055-afc034676c3f + jinja: 'Translate this into English: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -199,39 +213,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-source+target - reference: GPT-3 paper - 6b7cb084-c2cb-46ff-b432-72e754639497: !Template + - translate-ru-en-target + reference: '' + 9150eacb-0841-45ac-961d-6a95856c5803: !Template answer_choices: null - id: 6b7cb084-c2cb-46ff-b432-72e754639497 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 9150eacb-0841-45ac-961d-6a95856c5803 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - a_good_translation-en-ru-target reference: '' - 78994540-0af7-4b62-b90b-48e63ece9f69: !Template + a01d797c-29ff-45bc-b33f-1833692ae75b: !Template answer_choices: null - id: 78994540-0af7-4b62-b90b-48e63ece9f69 - jinja: 'Translate into English: {{translation["ru"]}} + id: a01d797c-29ff-45bc-b33f-1833692ae75b + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - version-ru-en-source+target reference: '' - 933bf831-8fee-4852-8fdf-4cc59670aaa0: !Template + a380ddce-9937-4235-87cf-d2e76658f590: !Template answer_choices: null - id: 933bf831-8fee-4852-8fdf-4cc59670aaa0 - jinja: 'How do you say {{translation["ru"]}} in English? + id: a380ddce-9937-4235-87cf-d2e76658f590 + jinja: 'Russian: {{translation["ru"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -240,11 +255,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target - reference: '' - 9a9e4d2b-0930-4940-ba9f-65bfcbcf835d: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ac8b312d-f909-4409-8453-3430b88c6bae: !Template answer_choices: null - id: 9a9e4d2b-0930-4940-ba9f-65bfcbcf835d + id: ac8b312d-f909-4409-8453-3430b88c6bae jinja: 'If the original version says: {{translation["en"]}}; then the Russian version should say: @@ -257,37 +272,37 @@ templates: name: !!python/tuple - version-en-ru-target reference: '' - 9b7040e1-2e67-4aaf-924f-ef69d08b3610: !Template + c6e3beed-6b46-487f-84e6-4621689fa107: !Template answer_choices: null - id: 9b7040e1-2e67-4aaf-924f-ef69d08b3610 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: c6e3beed-6b46-487f-84e6-4621689fa107 + jinja: 'How do you say {{translation["ru"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - how_to_say-ru-en-target reference: '' - ad2c10e3-8361-4177-bc43-a1bd88cfd592: !Template + db9616b7-d2d0-434a-90ee-26b8e27263dc: !Template answer_choices: null - id: ad2c10e3-8361-4177-bc43-a1bd88cfd592 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: db9616b7-d2d0-434a-90ee-26b8e27263dc + jinja: 'Translate this into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target + - translate-en-ru-target reference: '' - bac788df-2d7f-4133-a0a0-dac44d996160: !Template + df059410-48f4-4282-84dc-5d0724d63b25: !Template answer_choices: null - id: bac788df-2d7f-4133-a0a0-dac44d996160 + id: df059410-48f4-4282-84dc-5d0724d63b25 jinja: 'English: {{translation["en"]}} = Russian: ||| {{translation["ru"]}}' @@ -299,10 +314,10 @@ templates: name: !!python/tuple - xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - c45eb61c-01be-45fe-b0fe-3f6c987a063f: !Template + e40de616-dc50-44f4-859c-276a556c00a0: !Template answer_choices: null - id: c45eb61c-01be-45fe-b0fe-3f6c987a063f - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: e40de616-dc50-44f4-859c-276a556c00a0 + jinja: '{{translation["ru"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,12 +326,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target - reference: '' - e1d0c98c-cb14-4b97-977a-b869fe475006: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + eba2ae8b-ec7a-40c8-bb96-c74d8755cc03: !Template answer_choices: null - id: e1d0c98c-cb14-4b97-977a-b869fe475006 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: eba2ae8b-ec7a-40c8-bb96-c74d8755cc03 + jinja: 'Translate this from Russian into English: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -327,45 +342,30 @@ templates: name: !!python/tuple - translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - e82b2724-6d78-4013-a084-4e55102aecbe: !Template - answer_choices: null - id: e82b2724-6d78-4013-a084-4e55102aecbe - jinja: '{{translation["en"]}} = Russian: - - ||| {{translation["ru"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f39520c0-750f-4436-b116-9fbc3ada77c6: !Template + ef94f42c-dcab-40e0-9651-7e471a504320: !Template answer_choices: null - id: f39520c0-750f-4436-b116-9fbc3ada77c6 - jinja: 'What is the Russian translation of: {{translation["en"]}} - - ||| {{translation["ru"]}}' + id: ef94f42c-dcab-40e0-9651-7e471a504320 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: GPT-3 paper - fea25b62-6fb1-4512-8bba-cd9d88d83faf: !Template + - a_good_translation-ru-en-source+target + reference: '' + f200f945-6201-468b-b675-a6660f152c8f: !Template answer_choices: null - id: fea25b62-6fb1-4512-8bba-cd9d88d83faf - jinja: '{{translation["ru"]}} = English: + id: f200f945-6201-468b-b675-a6660f152c8f + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 + - translate_as_en-ru-target + reference: '' diff --git a/promptsource/templates/wmt17/tr-en/templates.yaml b/promptsource/templates/wmt17/tr-en/templates.yaml index fc09b2b06..e004575e9 100644 --- a/promptsource/templates/wmt17/tr-en/templates.yaml +++ b/promptsource/templates/wmt17/tr-en/templates.yaml @@ -1,307 +1,309 @@ dataset: wmt17 subset: tr-en templates: - 13e02590-6107-4130-973f-c1bde0b7761b: !Template + 0ad6ebf8-6256-4899-8014-480f096cd3e7: !Template answer_choices: null - id: 13e02590-6107-4130-973f-c1bde0b7761b - jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version - should say: + id: 0ad6ebf8-6256-4899-8014-480f096cd3e7 + jinja: ' {{translation["en"]}} translates into Turkish as: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-source+target + - translate_as_en-tr-target reference: '' - 18fa9dc2-e7b6-4724-8d90-1c14b2228a28: !Template + 13feec66-ea23-485c-833e-e75c1aa39c6b: !Template answer_choices: null - id: 18fa9dc2-e7b6-4724-8d90-1c14b2228a28 - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: 13feec66-ea23-485c-833e-e75c1aa39c6b + jinja: 'How do you say {{translation["en"]}} in Turkish? + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target + - how_to_say-en-tr-target reference: '' - 1efb1fbb-2b14-4923-9441-b91f79414748: !Template + 215d9910-906f-4e27-91a9-d4e2b10ca8c5: !Template answer_choices: null - id: 1efb1fbb-2b14-4923-9441-b91f79414748 - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: 215d9910-906f-4e27-91a9-d4e2b10ca8c5 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-source+starget + - version-tr-en-source+target reference: '' - 260eb8ab-1e49-434d-8c9a-1a4b32bec7c0: !Template + 2e3c2cbb-0069-44b0-8c5f-1dba0f82e52f: !Template answer_choices: null - id: 260eb8ab-1e49-434d-8c9a-1a4b32bec7c0 - jinja: 'Translate into Turkish: {{translation["en"]}} + id: 2e3c2cbb-0069-44b0-8c5f-1dba0f82e52f + jinja: 'Turkish: {{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-target - reference: '' - 273cf0ae-35da-493a-9850-384b9171b131: !Template + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3b6a8171-c577-46c9-a6c6-71f06347c296: !Template answer_choices: null - id: 273cf0ae-35da-493a-9850-384b9171b131 - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} + id: 3b6a8171-c577-46c9-a6c6-71f06347c296 + jinja: 'How do you say {{translation["tr"]}} in English? - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-source+target - reference: GPT 3 paper - 2990588c-38e5-476f-b0c5-feccda8b63da: !Template + - how_to_say-tr-en-target + reference: '' + 4508bd9c-1d22-4a29-bb03-270646863f54: !Template answer_choices: null - id: 2990588c-38e5-476f-b0c5-feccda8b63da - jinja: 'What is the Turkish translation of: {{translation["en"]}} + id: 4508bd9c-1d22-4a29-bb03-270646863f54 + jinja: 'What is the English translation of : {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target + - gpt-3-tr-en-target reference: GPT-3 paper - 2df1d1eb-efc4-4579-aa9a-40522ff6a765: !Template + 57019450-25a1-4af5-9264-c7c75f4426c1: !Template answer_choices: null - id: 2df1d1eb-efc4-4579-aa9a-40522ff6a765 - jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version - should say: + id: 57019450-25a1-4af5-9264-c7c75f4426c1 + jinja: 'Translate this into English: {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-source+target + - translate-tr-en-target reference: '' - 35a60dad-9931-4d04-b88d-c14832109928: !Template + 57f1aa08-5e65-48f2-9f2b-153ab8313120: !Template answer_choices: null - id: 35a60dad-9931-4d04-b88d-c14832109928 - jinja: '{{translation["tr"]}} translates into English as: + id: 57f1aa08-5e65-48f2-9f2b-153ab8313120 + jinja: 'Translate this into Turkish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-target + - translate-en-tr-target reference: '' - 39c782bb-130a-4d51-bc38-6f22e4993e78: !Template + 61123501-a3ff-469b-a952-11fd3a5ff6c5: !Template answer_choices: null - id: 39c782bb-130a-4d51-bc38-6f22e4993e78 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: 61123501-a3ff-469b-a952-11fd3a5ff6c5 + jinja: '{{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target - reference: '' - 44f6674b-965b-4bba-9b4e-b06588248248: !Template + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 6190b0e7-e1c5-48bd-8f1c-13634b044f69: !Template answer_choices: null - id: 44f6674b-965b-4bba-9b4e-b06588248248 - jinja: 'Turkish: {{translation["tr"]}} = English: + id: 6190b0e7-e1c5-48bd-8f1c-13634b044f69 + jinja: '{{translation["en"]}} = Turkish: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-source+target + - xglm-en-tr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 46aecf3a-1cfb-4736-b3bc-35eb6b69073e: !Template + 709c77c1-7710-4e53-ad0d-4ac623221e26: !Template answer_choices: null - id: 46aecf3a-1cfb-4736-b3bc-35eb6b69073e - jinja: 'What is the English translation of : {{translation["tr"]}} + id: 709c77c1-7710-4e53-ad0d-4ac623221e26 + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target - reference: GPT-3 paper - 4f537521-2acd-4fa5-9fa4-491959ae801d: !Template + - version-en-tr-source+target + reference: '' + 71a32989-8fd0-446f-b17e-eea66875a507: !Template answer_choices: null - id: 4f537521-2acd-4fa5-9fa4-491959ae801d - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: 71a32989-8fd0-446f-b17e-eea66875a507 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target + - a_good_translation-en-tr-source+target reference: '' - 5aab9b82-d8d6-4db5-a4c8-2c99dc86a822: !Template + 73623593-9c11-4348-8233-df90762b2e20: !Template answer_choices: null - id: 5aab9b82-d8d6-4db5-a4c8-2c99dc86a822 - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: 73623593-9c11-4348-8233-df90762b2e20 + jinja: 'What is the Turkish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target - reference: '' - 6be83853-7be0-48f8-a3b2-a27a2caa33bb: !Template + - gpt-3-en-tr-target + reference: GPT-3 paper + 7954790f-6fb4-4edb-b862-0549b3b22e58: !Template answer_choices: null - id: 6be83853-7be0-48f8-a3b2-a27a2caa33bb - jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 7954790f-6fb4-4edb-b862-0549b3b22e58 + jinja: 'English: {{translation["en"]}} = Turkish: + + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-source+target - reference: '' - 8a6bcf96-3f82-4673-973d-ff5b6b58ab7f: !Template + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 7a290d8f-87d0-480f-a65d-9e52c11d3fe6: !Template answer_choices: null - id: 8a6bcf96-3f82-4673-973d-ff5b6b58ab7f - jinja: ' {{translation["en"]}} translates into Turkish as: + id: 7a290d8f-87d0-480f-a65d-9e52c11d3fe6 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target + - version-tr-en-target reference: '' - 9da207fe-b01f-4fb2-a526-2dca9e72c040: !Template + 7c41a569-cd15-4d0f-b2c5-96146e853316: !Template answer_choices: null - id: 9da207fe-b01f-4fb2-a526-2dca9e72c040 - jinja: 'How do you say {{translation["tr"]}} in English? - - ||| {{translation["en"]}}' + id: 7c41a569-cd15-4d0f-b2c5-96146e853316 + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-tr-en-target + - a_good_translation-tr-en-source+target reference: '' - 9f258684-4a9a-4fd8-bbe8-8cf7dd204191: !Template + 9391038a-e0ed-49e0-8bbb-814b9afef7bc: !Template answer_choices: null - id: 9f258684-4a9a-4fd8-bbe8-8cf7dd204191 - jinja: 'Translate into English: {{translation["tr"]}} + id: 9391038a-e0ed-49e0-8bbb-814b9afef7bc + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-target - reference: '' - a0c2b202-34e9-4776-9171-da21c84aacea: !Template + - gpt-3-en-tr-source+target + reference: GPT 3 paper + 9e7bcfbc-73d6-41b5-9b9c-51d4be2455ed: !Template answer_choices: null - id: a0c2b202-34e9-4776-9171-da21c84aacea - jinja: 'English: {{translation["en"]}} = Turkish: + id: 9e7bcfbc-73d6-41b5-9b9c-51d4be2455ed + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - ad67e0db-d79c-475b-8e3c-14f31b5214e2: !Template + - a_good_translation-tr-en-target + reference: '' + ad102733-eee8-4ad7-9bb0-8a0629e02935: !Template answer_choices: null - id: ad67e0db-d79c-475b-8e3c-14f31b5214e2 - jinja: '{{translation["tr"]}} = English: + id: ad102733-eee8-4ad7-9bb0-8a0629e02935 + jinja: 'English: {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - b6be2a76-b9f2-4d73-9971-c92d4416c19c: !Template + - translate_as_en-tr-source+target + reference: '' + b0d7f2ab-fdf6-43ef-a34e-61778d68f2e7: !Template answer_choices: null - id: b6be2a76-b9f2-4d73-9971-c92d4416c19c - jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation - is: + id: b0d7f2ab-fdf6-43ef-a34e-61778d68f2e7 + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-target + - version-en-tr-target reference: '' - b80d257a-cd9c-4114-a3bd-d9c28aa2ab68: !Template + b554789a-45c1-4713-9788-c5e9789e36d0: !Template answer_choices: null - id: b80d257a-cd9c-4114-a3bd-d9c28aa2ab68 - jinja: '{{translation["en"]}} = Turkish: + id: b554789a-45c1-4713-9788-c5e9789e36d0 + jinja: 'Translate this from English into Turkish: {{translation["en"]}} - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d8bcf01b-6af0-4e6a-89a3-eb989436372f: !Template + - translate-en-tr-source+starget + reference: '' + b9272dc4-5f6e-462a-8c49-9bacb4843e0b: !Template answer_choices: null - id: d8bcf01b-6af0-4e6a-89a3-eb989436372f - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: b9272dc4-5f6e-462a-8c49-9bacb4843e0b + jinja: 'Translate this from Turkish into English: {{translation["tr"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -312,60 +314,58 @@ templates: name: !!python/tuple - translate-tr-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf - e655f136-c707-4313-a860-e3ab883baee7: !Template + bee2e383-cc8e-43a1-a317-ba7904a2b70a: !Template answer_choices: null - id: e655f136-c707-4313-a860-e3ab883baee7 - jinja: 'How do you say {{translation["en"]}} in Turkish? - - ||| {{translation["tr"]}}' + id: bee2e383-cc8e-43a1-a317-ba7904a2b70a + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - a_good_translation-en-tr-target reference: '' - ef26e5b8-fe37-4e05-b977-6dd89ac2a7b6: !Template + e23d2cf8-e1b0-4305-8326-ecd758a792e8: !Template answer_choices: null - id: ef26e5b8-fe37-4e05-b977-6dd89ac2a7b6 - jinja: 'English: {{translation["en"]}} translates into Turkish as: + id: e23d2cf8-e1b0-4305-8326-ecd758a792e8 + jinja: 'Turkish: {{translation["tr"]}} translates into English as: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target + - translate_as_tr-en-source+target reference: '' - f8ba7995-30e2-4b90-9e6c-da6058983479: !Template + f4601284-1c19-4c50-98d1-d1be4ef142ce: !Template answer_choices: null - id: f8ba7995-30e2-4b90-9e6c-da6058983479 - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: f4601284-1c19-4c50-98d1-d1be4ef142ce + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target - reference: '' - fd327d86-96c0-4a0e-ab1c-ae1110398d11: !Template + - gpt-3-tr-en-source+target + reference: GPT-3 paper + fe200cad-1f06-4805-943a-88355d1ce6e2: !Template answer_choices: null - id: fd327d86-96c0-4a0e-ab1c-ae1110398d11 - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: fe200cad-1f06-4805-943a-88355d1ce6e2 + jinja: '{{translation["tr"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-source+target - reference: GPT-3 paper + - translate_as_tr-en-target + reference: '' diff --git a/promptsource/templates/wmt17/zh-en/templates.yaml b/promptsource/templates/wmt17/zh-en/templates.yaml index 946b2e007..14a99458f 100644 --- a/promptsource/templates/wmt17/zh-en/templates.yaml +++ b/promptsource/templates/wmt17/zh-en/templates.yaml @@ -1,25 +1,23 @@ dataset: wmt17 subset: zh-en templates: - 056b54ef-6044-4fdc-997a-94e7d70c0ae6: !Template + 01c86605-b157-42a3-acde-419673e43fab: !Template answer_choices: null - id: 056b54ef-6044-4fdc-997a-94e7d70c0ae6 - jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: 01c86605-b157-42a3-acde-419673e43fab + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-target + - a_good_translation-zh-en-source+target reference: '' - 1183a3f9-7303-488b-9273-b92e4dc767f0: !Template + 1022f169-5b09-4efc-bb8c-c80b1476c6ea: !Template answer_choices: null - id: 1183a3f9-7303-488b-9273-b92e4dc767f0 - jinja: '{{translation["en"]}} = Chinese: + id: 1022f169-5b09-4efc-bb8c-c80b1476c6ea + jinja: 'English: {{translation["en"]}} = Chinese: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -28,110 +26,113 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 169b309d-5851-4444-a208-5bb91cde3af3: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 24c02b50-8aec-4645-91ba-e6b74b4659ce: !Template answer_choices: null - id: 169b309d-5851-4444-a208-5bb91cde3af3 - jinja: 'How do you say {{translation["zh"]}} in English? + id: 24c02b50-8aec-4645-91ba-e6b74b4659ce + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target + - version-en-zh-source+target reference: '' - 1c810691-6c34-4e80-87ce-653ddb7330c9: !Template + 2ef8654a-9cce-4602-88f5-f56f1b8d5d88: !Template answer_choices: null - id: 1c810691-6c34-4e80-87ce-653ddb7330c9 - jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation - is: ||| {{translation["zh"]}}' + id: 2ef8654a-9cce-4602-88f5-f56f1b8d5d88 + jinja: 'Translate this from English into Chinese: {{translation["en"]}} + + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-target + - translate-en-zh-source+starget reference: '' - 204091fc-5795-4dea-a245-0816eeb5c525: !Template + 38349a24-8617-4ce2-ac6b-6e399fa5f926: !Template answer_choices: null - id: 204091fc-5795-4dea-a245-0816eeb5c525 - jinja: 'Chinese: {{translation["zh"]}} = English: + id: 38349a24-8617-4ce2-ac6b-6e399fa5f926 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 21880c27-4a9e-4676-a4c3-3685b4ea9d7d: !Template + - version-zh-en-source+target + reference: '' + 3a8705c6-a21d-47e3-917c-1eb50961af15: !Template answer_choices: null - id: 21880c27-4a9e-4676-a4c3-3685b4ea9d7d - jinja: 'English: {{translation["en"]}} = Chinese: + id: 3a8705c6-a21d-47e3-917c-1eb50961af15 + jinja: 'Translate this from Chinese into English: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 30b7a571-9cb1-458f-bdb7-90eb442d0f15: !Template + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 3b3240ec-ddc6-4f46-84c3-da784888db1c: !Template answer_choices: null - id: 30b7a571-9cb1-458f-bdb7-90eb442d0f15 - jinja: 'Translate into English: {{translation["zh"]}} + id: 3b3240ec-ddc6-4f46-84c3-da784888db1c + jinja: ' {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-target + - translate_as_en-zh-target reference: '' - 486bb7ac-1d82-4f8c-8795-0fdcbefdc44c: !Template + 43f47446-52d2-41a3-9ca3-abbb3fc1a93f: !Template answer_choices: null - id: 486bb7ac-1d82-4f8c-8795-0fdcbefdc44c - jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version - should say: + id: 43f47446-52d2-41a3-9ca3-abbb3fc1a93f + jinja: 'Chinese: {{translation["zh"]}} = English: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-source+target - reference: '' - 4b000b4b-64e0-4bc1-947d-3f7e967d8116: !Template + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 45a91ef1-00d9-474f-a945-c2acfaee245b: !Template answer_choices: null - id: 4b000b4b-64e0-4bc1-947d-3f7e967d8116 - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: 45a91ef1-00d9-474f-a945-c2acfaee245b + jinja: 'What is the English translation of : {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target - reference: '' - 5821cf3f-41f2-4144-adc5-e6322e47e018: !Template + - gpt-3-zh-en-target + reference: GPT-3 paper + 55efd980-0439-4eee-9ec9-bb8baf07bf8a: !Template answer_choices: null - id: 5821cf3f-41f2-4144-adc5-e6322e47e018 - jinja: ' {{translation["en"]}} translates into Chinese as: + id: 55efd980-0439-4eee-9ec9-bb8baf07bf8a + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -140,111 +141,109 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-target + - a_good_translation-en-zh-source+target reference: '' - 5cdb4bca-fa23-4637-9459-02880508ec89: !Template + 6696796b-d19b-4cba-a840-18b234522e59: !Template answer_choices: null - id: 5cdb4bca-fa23-4637-9459-02880508ec89 - jinja: '{{translation["zh"]}} translates into English as: + id: 6696796b-d19b-4cba-a840-18b234522e59 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target - reference: '' - 7aff9be8-0ffd-47a1-aeb8-3c77daffcc33: !Template + - gpt-3-en-zh-source+target + reference: GPT 3 paper + 6b31d70d-c109-44a6-aae8-22918c41127d: !Template answer_choices: null - id: 7aff9be8-0ffd-47a1-aeb8-3c77daffcc33 - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: 6b31d70d-c109-44a6-aae8-22918c41127d + jinja: '{{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target - reference: '' - 82897041-28e7-43d6-a9cc-4adf2599dda4: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 7c49b8a1-82cb-4ad3-96d2-be71e7bf4401: !Template answer_choices: null - id: 82897041-28e7-43d6-a9cc-4adf2599dda4 - jinja: 'Translate Chinese into English: {{translation["zh"]}} + id: 7c49b8a1-82cb-4ad3-96d2-be71e7bf4401 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 877c7d17-b3bc-4637-ac86-d0903d8bd4fe: !Template + - version-zh-en-target + reference: '' + 8b8653bb-d13a-4ad6-8236-d851b2036861: !Template answer_choices: null - id: 877c7d17-b3bc-4637-ac86-d0903d8bd4fe - jinja: 'What is the Chinese translation of: {{translation["en"]}} + id: 8b8653bb-d13a-4ad6-8236-d851b2036861 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-target - reference: GPT-3 paper - 8cdeef19-7f5d-477d-b4a7-6859f01fb581: !Template + - translate_as_zh-en-source+target + reference: '' + 909a8bb9-3c03-4a86-bd16-de054ce83f73: !Template answer_choices: null - id: 8cdeef19-7f5d-477d-b4a7-6859f01fb581 - jinja: '{{translation["zh"]}} = English: + id: 909a8bb9-3c03-4a86-bd16-de054ce83f73 + jinja: 'Translate this into Chinese: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - a3d7aab6-d5a6-4a93-b516-bfe3a7147228: !Template + - translate-en-zh-target + reference: '' + 9798947f-7a8e-4e1b-aa3a-8c51df5c761b: !Template answer_choices: null - id: a3d7aab6-d5a6-4a93-b516-bfe3a7147228 - jinja: 'Translate English into Chinese: {{translation["en"]}} + id: 9798947f-7a8e-4e1b-aa3a-8c51df5c761b + jinja: '{{translation["zh"]}} translates into English as: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-source+starget + - translate_as_zh-en-target reference: '' - ac3f9e03-e3bb-438a-b4e8-89e369dcc998: !Template + b4302b25-ec16-49e3-bbf5-f994b73c9108: !Template answer_choices: null - id: ac3f9e03-e3bb-438a-b4e8-89e369dcc998 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Chinese translation is: - - ||| {{translation["zh"]}}' + id: b4302b25-ec16-49e3-bbf5-f994b73c9108 + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-source+target + - a_good_translation-en-zh-target reference: '' - afb387ba-0dd6-4ba8-8b74-4fdeaeeaba46: !Template + b6b0428e-e988-4408-9cea-5de22a57006d: !Template answer_choices: null - id: afb387ba-0dd6-4ba8-8b74-4fdeaeeaba46 + id: b6b0428e-e988-4408-9cea-5de22a57006d jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} ||| {{translation["en"]}}' @@ -256,116 +255,117 @@ templates: name: !!python/tuple - gpt-3-zh-en-source+target reference: GPT-3 paper - c2ae991d-4700-433c-bf62-4cc0f69e7d35: !Template + cca404b5-2af7-493a-a31d-97c0417d1b28: !Template answer_choices: null - id: c2ae991d-4700-433c-bf62-4cc0f69e7d35 - jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: cca404b5-2af7-493a-a31d-97c0417d1b28 + jinja: 'How do you say {{translation["zh"]}} in English? + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-source+target + - how_to_say-zh-en-target reference: '' - c4d02e4e-1887-4ba9-8adc-80f7fe693ae2: !Template + d9987f06-c787-45d0-b05f-3b6857f2fcb7: !Template answer_choices: null - id: c4d02e4e-1887-4ba9-8adc-80f7fe693ae2 - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: d9987f06-c787-45d0-b05f-3b6857f2fcb7 + jinja: '{{translation["zh"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target - reference: '' - c61d619c-6f9a-46be-bf7b-205f6f36ab00: !Template + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + dd529993-6d65-497f-9cea-ba3104763099: !Template answer_choices: null - id: c61d619c-6f9a-46be-bf7b-205f6f36ab00 - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese - version should say: + id: dd529993-6d65-497f-9cea-ba3104763099 + jinja: 'What is the Chinese translation of: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target - reference: '' - c6c6cc52-2295-4782-8b34-c4d169cdc771: !Template + - gpt-3-en-zh-target + reference: GPT-3 paper + e2d3e03b-ee52-4819-a8f9-c97807ab15ae: !Template answer_choices: null - id: c6c6cc52-2295-4782-8b34-c4d169cdc771 - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: e2d3e03b-ee52-4819-a8f9-c97807ab15ae + jinja: 'How do you say {{translation["en"]}} in Chinese? - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-source+target - reference: GPT 3 paper - d8779d0f-5d92-4873-bb34-3782cf31d47d: !Template + - how_to_say-en-zh-target + reference: '' + e47899c3-8ec9-41a0-b78e-1379e8a04d61: !Template answer_choices: null - id: d8779d0f-5d92-4873-bb34-3782cf31d47d - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: e47899c3-8ec9-41a0-b78e-1379e8a04d61 + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target + - a_good_translation-zh-en-target reference: '' - ea8d50d2-ffdf-4f3b-9f0a-fdb4dd9834ba: !Template + f067f264-a065-466d-ba1a-90b88b6927bd: !Template answer_choices: null - id: ea8d50d2-ffdf-4f3b-9f0a-fdb4dd9834ba - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: f067f264-a065-466d-ba1a-90b88b6927bd + jinja: 'Translate this into English: {{translation["zh"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target + - translate-zh-en-target reference: '' - eed08d24-e22a-4a29-8336-fc58912dc806: !Template + f0788fdb-f012-40d4-b5ff-f1a1e17472e8: !Template answer_choices: null - id: eed08d24-e22a-4a29-8336-fc58912dc806 - jinja: 'What is the English translation of : {{translation["zh"]}} + id: f0788fdb-f012-40d4-b5ff-f1a1e17472e8 + jinja: 'English: {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-target - reference: GPT-3 paper - fdbe9566-0482-4697-8d71-aae344ce4d5d: !Template + - translate_as_en-zh-source+target + reference: '' + f51d1d6f-0555-43fb-b00a-ae7800153d1a: !Template answer_choices: null - id: fdbe9566-0482-4697-8d71-aae344ce4d5d - jinja: 'Translate into Chinese: {{translation["en"]}} + id: f51d1d6f-0555-43fb-b00a-ae7800153d1a + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - version-en-zh-target reference: '' diff --git a/promptsource/templates/wmt18/cs-en/templates.yaml b/promptsource/templates/wmt18/cs-en/templates.yaml index a0cd89cac..1b004aaeb 100644 --- a/promptsource/templates/wmt18/cs-en/templates.yaml +++ b/promptsource/templates/wmt18/cs-en/templates.yaml @@ -1,9 +1,9 @@ dataset: wmt18 subset: cs-en templates: - 04577e1a-ffdf-4b1e-83d2-b0e904710680: !Template + 03327d86-02e1-4104-8ae2-b435fe7408f8: !Template answer_choices: null - id: 04577e1a-ffdf-4b1e-83d2-b0e904710680 + id: 03327d86-02e1-4104-8ae2-b435fe7408f8 jinja: 'What is the English translation of : {{translation["cs"]}} ||| {{translation["en"]}}' @@ -15,10 +15,10 @@ templates: name: !!python/tuple - gpt-3-cs-en-target reference: GPT-3 paper - 05776b12-0456-4600-966f-9faf823093ee: !Template + 08968ba9-7655-4c6f-b944-15a79c3532cd: !Template answer_choices: null - id: 05776b12-0456-4600-966f-9faf823093ee - jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} + id: 08968ba9-7655-4c6f-b944-15a79c3532cd + jinja: 'Translate this from Czech into English: {{translation["cs"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -27,27 +27,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-cs-en-source+target - reference: GPT-3 paper - 058e1721-21a8-4ce0-a2bf-6880d2bf8e29: !Template + - translate-cs-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 096afb1d-46ea-43b6-931e-2ec156616532: !Template answer_choices: null - id: 058e1721-21a8-4ce0-a2bf-6880d2bf8e29 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Czech translation is: + id: 096afb1d-46ea-43b6-931e-2ec156616532 + jinja: 'How do you say {{translation["cs"]}} in English? - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-source+target + - how_to_say-cs-en-target reference: '' - 05923706-77be-4421-9f2d-bd4508615e37: !Template + 0cd65c6f-88c2-47d9-890d-eebca06c04b5: !Template answer_choices: null - id: 05923706-77be-4421-9f2d-bd4508615e37 - jinja: ' {{translation["en"]}} translates into Czech as: + id: 0cd65c6f-88c2-47d9-890d-eebca06c04b5 + jinja: 'English: {{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -56,55 +55,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-target - reference: '' - 24252dfb-1e81-4c70-b482-f4c6efbbe6d3: !Template + - xglm-en-cs-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 1437cd13-5350-4b01-8726-4dff81690098: !Template answer_choices: null - id: 24252dfb-1e81-4c70-b482-f4c6efbbe6d3 - jinja: 'How do you say {{translation["en"]}} in Czech? + id: 1437cd13-5350-4b01-8726-4dff81690098 + jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation + is: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-cs-target + - a_good_translation-cs-en-target reference: '' - 3e7e4a69-d624-4d0b-909b-2e212feafa75: !Template + 27666a03-bc94-49a8-a4e7-59f43402ac1f: !Template answer_choices: null - id: 3e7e4a69-d624-4d0b-909b-2e212feafa75 - jinja: 'If the English version says: {{translation["en"]}}; then the Czech version - should say: + id: 27666a03-bc94-49a8-a4e7-59f43402ac1f + jinja: 'Czech: {{translation["cs"]}} translates into English as: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-cs-source+target + - translate_as_cs-en-source+target reference: '' - 49acd099-d766-48fc-8849-bb7620b6960d: !Template + 33725eb8-5f79-408f-9d7a-1fedb11d0aef: !Template answer_choices: null - id: 49acd099-d766-48fc-8849-bb7620b6960d - jinja: 'How do you say {{translation["cs"]}} in English? + id: 33725eb8-5f79-408f-9d7a-1fedb11d0aef + jinja: 'Translate this from English into Czech: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-cs-en-target + - translate-en-cs-source+starget reference: '' - 5e1da87f-b520-4ce6-9663-b22b546eed9c: !Template + 34a73c3b-91ea-4995-b183-adf475db0006: !Template answer_choices: null - id: 5e1da87f-b520-4ce6-9663-b22b546eed9c - jinja: 'Czech: {{translation["cs"]}} = English: + id: 34a73c3b-91ea-4995-b183-adf475db0006 + jinja: '{{translation["cs"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -113,81 +112,80 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 7d3c501c-5c7c-4e54-84e1-97b24e90e673: !Template + - translate_as_cs-en-target + reference: '' + 3afe3c3a-8f80-4b2c-b0d4-1f53e288fc0e: !Template answer_choices: null - id: 7d3c501c-5c7c-4e54-84e1-97b24e90e673 - jinja: 'English: {{translation["en"]}} translates into Czech as: + id: 3afe3c3a-8f80-4b2c-b0d4-1f53e288fc0e + jinja: 'Translate this into English: {{translation["cs"]}} - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-cs-source+target + - translate-cs-en-target reference: '' - 806dc491-5193-42ce-a783-2801eef5b36b: !Template + 4087a143-71a4-4174-89c2-af3836c3fbf1: !Template answer_choices: null - id: 806dc491-5193-42ce-a783-2801eef5b36b - jinja: 'Translate English into Czech: {{translation["en"]}} + id: 4087a143-71a4-4174-89c2-af3836c3fbf1 + jinja: 'Czech: {{translation["cs"]}} = English: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-cs-source+starget - reference: '' - 86b123b4-5185-4dbe-9f49-30eb4b3181e0: !Template + - xglm-cs-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4a5edbb5-7647-48d5-8ffb-c99d39de35ab: !Template answer_choices: null - id: 86b123b4-5185-4dbe-9f49-30eb4b3181e0 - jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 4a5edbb5-7647-48d5-8ffb-c99d39de35ab + jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation + is: ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-source+target + - a_good_translation-en-cs-target reference: '' - 92d63f28-f816-4432-a151-216f972297de: !Template + 6676a4d3-0621-43bb-bb97-269c3f3718f7: !Template answer_choices: null - id: 92d63f28-f816-4432-a151-216f972297de - jinja: '{{translation["cs"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 6676a4d3-0621-43bb-bb97-269c3f3718f7 + jinja: 'Given the following source text in Czech: {{translation["cs"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-target + - a_good_translation-cs-en-source+target reference: '' - 9953d68f-23d5-484f-a1c1-d714e2b529d4: !Template + 7713da9d-ccdd-43ce-9895-503afee2bb8a: !Template answer_choices: null - id: 9953d68f-23d5-484f-a1c1-d714e2b529d4 - jinja: 'Translate Czech into English: {{translation["cs"]}} + id: 7713da9d-ccdd-43ce-9895-503afee2bb8a + jinja: 'How do you say {{translation["en"]}} in Czech? - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - a33509f7-bd83-44c4-8335-8e05cd74c0f5: !Template + - how_to_say-en-cs-target + reference: '' + 82200d10-5851-4f29-80f6-42392ab794a9: !Template answer_choices: null - id: a33509f7-bd83-44c4-8335-8e05cd74c0f5 - jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} + id: 82200d10-5851-4f29-80f6-42392ab794a9 + jinja: 'What is the Czech translation of: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -196,26 +194,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-cs-source+target - reference: GPT 3 paper - b074fa3f-a7ca-4d09-982d-28279d7567a7: !Template + - gpt-3-en-cs-target + reference: GPT-3 paper + 838161af-dd29-463c-8561-f016f2f7054b: !Template answer_choices: null - id: b074fa3f-a7ca-4d09-982d-28279d7567a7 - jinja: '{{translation["en"]}} = Czech: + id: 838161af-dd29-463c-8561-f016f2f7054b + jinja: 'If the original version says: {{translation["cs"]}}; then the English + version should say: - ||| {{translation["cs"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - b0d50e7e-b106-4a3b-9ec1-d4d3e4a6be50: !Template + - version-cs-en-target + reference: '' + 91e38fee-b424-47bc-a8c3-b47eb4cf1624: !Template answer_choices: null - id: b0d50e7e-b106-4a3b-9ec1-d4d3e4a6be50 - jinja: 'Translate into Czech: {{translation["en"]}} + id: 91e38fee-b424-47bc-a8c3-b47eb4cf1624 + jinja: 'Translate this into Czech: {{translation["en"]}} ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -226,23 +225,9 @@ templates: name: !!python/tuple - translate-en-cs-target reference: '' - b1a91890-7da9-45b5-a9e1-77a5b22411b8: !Template - answer_choices: null - id: b1a91890-7da9-45b5-a9e1-77a5b22411b8 - jinja: 'What is the Czech translation of: {{translation["en"]}} - - ||| {{translation["cs"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - gpt-3-en-cs-target - reference: GPT-3 paper - b64106c0-74a9-4860-87ea-b1046f85601f: !Template + 93d9e3cd-34ea-45f7-a2be-0452f7eec9dd: !Template answer_choices: null - id: b64106c0-74a9-4860-87ea-b1046f85601f + id: 93d9e3cd-34ea-45f7-a2be-0452f7eec9dd jinja: 'If the original version says: {{translation["en"]}}; then the Czech version should say: @@ -255,95 +240,95 @@ templates: name: !!python/tuple - version-en-cs-target reference: '' - b742c3a5-1276-4992-a84f-f1f225d6f406: !Template + 9cea0cbb-aa40-4e91-bdb9-3d926c647cbe: !Template answer_choices: null - id: b742c3a5-1276-4992-a84f-f1f225d6f406 - jinja: 'Given the following passage: {{translation["en"]}} , a good Czech translation - is: ||| {{translation["cs"]}}' + id: 9cea0cbb-aa40-4e91-bdb9-3d926c647cbe + jinja: '{{translation["cs"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-cs-target - reference: '' - c4462899-417c-4975-9dbe-e63a6de9f340: !Template + - xglm-cs-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + a572a3b3-9ef4-482e-89b7-bc248231f75b: !Template answer_choices: null - id: c4462899-417c-4975-9dbe-e63a6de9f340 - jinja: 'Czech: {{translation["cs"]}} translates into English as: + id: a572a3b3-9ef4-482e-89b7-bc248231f75b + jinja: 'English: {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_cs-en-source+target + - translate_as_en-cs-source+target reference: '' - cc1a6c32-2996-4106-92f8-50ddb3b67c4d: !Template + a985c6f8-4a7f-4cba-a407-1931b048a3a3: !Template answer_choices: null - id: cc1a6c32-2996-4106-92f8-50ddb3b67c4d - jinja: '{{translation["cs"]}} = English: + id: a985c6f8-4a7f-4cba-a407-1931b048a3a3 + jinja: 'What is the English translation of the Czech sentence: {{translation["cs"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-cs-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - d434a2cd-8da4-4099-9a5d-f213adb25f0a: !Template + - gpt-3-cs-en-source+target + reference: GPT-3 paper + b3545af0-9c6e-49fa-beb3-40bcbda3128f: !Template answer_choices: null - id: d434a2cd-8da4-4099-9a5d-f213adb25f0a - jinja: 'Given the following passage: {{translation["cs"]}} , a good English translation - is: + id: b3545af0-9c6e-49fa-beb3-40bcbda3128f + jinja: 'If the English version says: {{translation["en"]}}; then the Czech version + should say: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-cs-en-target + - version-en-cs-source+target reference: '' - da493ed1-90a4-4611-a423-a86610ffae8f: !Template + bbb3bf9c-ce66-490e-abbc-b3166600d813: !Template answer_choices: null - id: da493ed1-90a4-4611-a423-a86610ffae8f - jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version - should say: + id: bbb3bf9c-ce66-490e-abbc-b3166600d813 + jinja: 'What is the Czech translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-cs-en-source+target - reference: '' - e28156c4-1a78-4240-b8f7-1c89dee1e938: !Template + - gpt-3-en-cs-source+target + reference: GPT 3 paper + c506feaa-fe44-4ddf-83c2-059e36811105: !Template answer_choices: null - id: e28156c4-1a78-4240-b8f7-1c89dee1e938 - jinja: 'Translate into English: {{translation["cs"]}} + id: c506feaa-fe44-4ddf-83c2-059e36811105 + jinja: ' {{translation["en"]}} translates into Czech as: - ||| {{translation["en"]}}' + ||| {{translation["cs"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-cs-en-target + - translate_as_en-cs-target reference: '' - f1f6a13a-3038-4e85-abfb-e110ef02a2b2: !Template + cc4fc133-d6e1-4684-9bdd-b20fc22be63f: !Template answer_choices: null - id: f1f6a13a-3038-4e85-abfb-e110ef02a2b2 - jinja: 'English: {{translation["en"]}} = Czech: + id: cc4fc133-d6e1-4684-9bdd-b20fc22be63f + jinja: '{{translation["en"]}} = Czech: ||| {{translation["cs"]}}' metadata: !TemplateMetadata @@ -352,13 +337,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-cs-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - f804109f-5858-4d43-868b-d5d6304c8146: !Template + - xglm-en-cs-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + ce04a4bb-7ea4-40b4-aac9-c268f9fecf3c: !Template answer_choices: null - id: f804109f-5858-4d43-868b-d5d6304c8146 - jinja: 'If the original version says: {{translation["cs"]}}; then the English - version should say: + id: ce04a4bb-7ea4-40b4-aac9-c268f9fecf3c + jinja: 'If the Czech version says: {{translation["cs"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -367,5 +352,20 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-cs-en-target + - version-cs-en-source+target + reference: '' + dce3d790-25d0-4b36-b47b-bc9935859cd8: !Template + answer_choices: null + id: dce3d790-25d0-4b36-b47b-bc9935859cd8 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Czech translation is: + + ||| {{translation["cs"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-cs-source+target reference: '' diff --git a/promptsource/templates/wmt18/de-en/templates.yaml b/promptsource/templates/wmt18/de-en/templates.yaml index 3c1007ec8..ca3110abd 100644 --- a/promptsource/templates/wmt18/de-en/templates.yaml +++ b/promptsource/templates/wmt18/de-en/templates.yaml @@ -1,25 +1,10 @@ dataset: wmt18 subset: de-en templates: - 07ad9f5e-0c3e-4eec-9381-533cfe063087: !Template + 02c614c3-4674-48ea-8c0a-6bbb17bf266c: !Template answer_choices: null - id: 07ad9f5e-0c3e-4eec-9381-533cfe063087 - jinja: 'German: {{translation["de"]}} translates into English as: - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - translate_as_de-en-source+target - reference: '' - 0a9966ee-ce22-464e-bb0e-3353d357fde4: !Template - answer_choices: null - id: 0a9966ee-ce22-464e-bb0e-3353d357fde4 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 02c614c3-4674-48ea-8c0a-6bbb17bf266c + jinja: 'English: {{translation["en"]}} translates into German as: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -28,26 +13,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target - reference: '' - 17e768cb-2d12-458b-822c-8b3566e0b512: !Template - answer_choices: null - id: 17e768cb-2d12-458b-822c-8b3566e0b512 - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: - - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - version-en-de-target + - translate_as_en-de-source+target reference: '' - 21963372-77d8-46f6-9514-ea839e998a17: !Template + 0d2188af-dc45-476b-9a6d-e3ecc53aa5cd: !Template answer_choices: null - id: 21963372-77d8-46f6-9514-ea839e998a17 + id: 0d2188af-dc45-476b-9a6d-e3ecc53aa5cd jinja: 'How do you say {{translation["en"]}} in German? ||| {{translation["de"]}}' @@ -59,10 +29,10 @@ templates: name: !!python/tuple - how_to_say-en-de-target reference: '' - 260e9766-9de6-4c2c-85a4-6b803b18ceaf: !Template + 12205792-1352-4ad8-a85a-7e3929022c8b: !Template answer_choices: null - id: 260e9766-9de6-4c2c-85a4-6b803b18ceaf - jinja: 'Translate into German: {{translation["en"]}} + id: 12205792-1352-4ad8-a85a-7e3929022c8b + jinja: 'Translate this from English into German: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -71,26 +41,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target + - translate-en-de-source+starget reference: '' - 285711ca-7f22-4e81-b56a-d909f3bd0467: !Template + 28604275-67d1-4231-ac6a-73692dbc66c4: !Template answer_choices: null - id: 285711ca-7f22-4e81-b56a-d909f3bd0467 - jinja: 'If the German version says: {{translation["de"]}}; then the English version - should say: + id: 28604275-67d1-4231-ac6a-73692dbc66c4 + jinja: 'Translate this into German: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - translate-en-de-target reference: '' - 2ca87d75-82bd-47bb-8f35-5330c2357bd0: !Template + 48fff56c-0bec-4dbd-bf09-2a5e98a77219: !Template answer_choices: null - id: 2ca87d75-82bd-47bb-8f35-5330c2357bd0 + id: 48fff56c-0bec-4dbd-bf09-2a5e98a77219 jinja: ' {{translation["en"]}} translates into German as: ||| {{translation["de"]}}' @@ -102,52 +71,52 @@ templates: name: !!python/tuple - translate_as_en-de-target reference: '' - 3df26f11-69f5-4c35-bc41-2a2088cc3f5c: !Template + 55c03738-d30d-4ce0-a3f9-ba548580351a: !Template answer_choices: null - id: 3df26f11-69f5-4c35-bc41-2a2088cc3f5c - jinja: 'English: {{translation["en"]}} translates into German as: + id: 55c03738-d30d-4ce0-a3f9-ba548580351a + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - a_good_translation-de-en-target reference: '' - 56ba2774-5301-47b3-a727-c15ab3e57d98: !Template + 5fa73a35-d47d-45ff-9457-6664066b01c4: !Template answer_choices: null - id: 56ba2774-5301-47b3-a727-c15ab3e57d98 - jinja: 'What is the English translation of : {{translation["de"]}} - - ||| {{translation["en"]}}' + id: 5fa73a35-d47d-45ff-9457-6664066b01c4 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target - reference: GPT-3 paper - 5a1aa7e3-4979-4e92-b9ae-5d24c9a7d8b7: !Template + - a_good_translation-de-en-source+target + reference: '' + 626b674e-c57d-452b-bfd9-6523520542e1: !Template answer_choices: null - id: 5a1aa7e3-4979-4e92-b9ae-5d24c9a7d8b7 - jinja: '{{translation["de"]}} = English: + id: 626b674e-c57d-452b-bfd9-6523520542e1 + jinja: 'What is the German translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 6250fa75-6aa8-4942-a911-e858912a81fe: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + 67e8b71d-e37d-4336-a354-48c769f9a591: !Template answer_choices: null - id: 6250fa75-6aa8-4942-a911-e858912a81fe - jinja: 'German: {{translation["de"]}} = English: + id: 67e8b71d-e37d-4336-a354-48c769f9a591 + jinja: '{{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -156,13 +125,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-source+target + - xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 6712c4ec-94f8-48e5-8766-65b99d24a2e0: !Template + 67f3ac0a-3ae1-4a7f-9a0e-0ef1619298aa: !Template answer_choices: null - id: 6712c4ec-94f8-48e5-8766-65b99d24a2e0 - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: + id: 67f3ac0a-3ae1-4a7f-9a0e-0ef1619298aa + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -171,13 +140,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - version-de-en-source+target reference: '' - 71e66088-b150-4465-9912-01177de1a4ad: !Template + 70965178-8a49-47e7-8956-1c6d081eb090: !Template answer_choices: null - id: 71e66088-b150-4465-9912-01177de1a4ad - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 70965178-8a49-47e7-8956-1c6d081eb090 + jinja: 'How do you say {{translation["de"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -186,53 +154,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - how_to_say-de-en-target reference: '' - 77063960-efd8-43a8-b9f4-171309efa924: !Template + 746f42a6-bdd3-4cae-b8da-5301f515c650: !Template answer_choices: null - id: 77063960-efd8-43a8-b9f4-171309efa924 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 746f42a6-bdd3-4cae-b8da-5301f515c650 + jinja: 'What is the English translation of : {{translation["de"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target - reference: '' - 78385c61-e86d-4838-a5df-47d2d41a44f6: !Template + - gpt-3-de-en-target + reference: GPT-3 paper + 75010174-babf-4c15-b21b-307f3ae1d739: !Template answer_choices: null - id: 78385c61-e86d-4838-a5df-47d2d41a44f6 - jinja: '{{translation["de"]}} translates into English as: + id: 75010174-babf-4c15-b21b-307f3ae1d739 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - a_good_translation-en-de-source+target reference: '' - 789c7529-7b54-4ca5-8d41-860f00ec6e12: !Template + 76fed237-5a46-4494-9d0c-c4158b046aa0: !Template answer_choices: null - id: 789c7529-7b54-4ca5-8d41-860f00ec6e12 - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 76fed237-5a46-4494-9d0c-c4158b046aa0 + jinja: '{{translation["en"]}} = German: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target - reference: GPT-3 paper - 871d391f-641c-4278-a6d2-42cebc56f3f7: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 815753ac-61e0-4090-9298-b29e04244953: !Template answer_choices: null - id: 871d391f-641c-4278-a6d2-42cebc56f3f7 - jinja: '{{translation["en"]}} = German: + id: 815753ac-61e0-4090-9298-b29e04244953 + jinja: 'English: {{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -241,124 +211,126 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 8f46d8fd-1d54-49d0-bce4-805cb3cd36bb: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 8d17685a-52c3-4600-a5f6-98f949e358bc: !Template answer_choices: null - id: 8f46d8fd-1d54-49d0-bce4-805cb3cd36bb - jinja: 'What is the German translation of: {{translation["en"]}} + id: 8d17685a-52c3-4600-a5f6-98f949e358bc + jinja: 'If the English version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: GPT-3 paper - a8fa13e1-eb34-410b-9d50-c353a24c26b5: !Template + - version-en-de-source+target + reference: '' + 8ed9dc8c-10ae-4744-8f80-c07a3b267602: !Template answer_choices: null - id: a8fa13e1-eb34-410b-9d50-c353a24c26b5 - jinja: 'How do you say {{translation["de"]}} in English? + id: 8ed9dc8c-10ae-4744-8f80-c07a3b267602 + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target + - version-en-de-target reference: '' - b86dadd8-d601-47d6-820c-3d65fb1db513: !Template + 91fe5fef-9f0a-496f-8bc0-f04043c1bc0f: !Template answer_choices: null - id: b86dadd8-d601-47d6-820c-3d65fb1db513 - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: 91fe5fef-9f0a-496f-8bc0-f04043c1bc0f + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-source+target - reference: GPT 3 paper - c2860cd8-3358-4109-b07f-2b533c9224b8: !Template + - version-de-en-target + reference: '' + 9780d39d-4d81-4dfa-881a-7380f19a47fa: !Template answer_choices: null - id: c2860cd8-3358-4109-b07f-2b533c9224b8 - jinja: 'English: {{translation["en"]}} = German: + id: 9780d39d-4d81-4dfa-881a-7380f19a47fa + jinja: 'Translate this into English: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - d522ed8d-9b4b-44cc-b4c6-d966673eef3f: !Template + - translate-de-en-target + reference: '' + b5866f82-9db7-4b89-9b6f-42168dcf4113: !Template answer_choices: null - id: d522ed8d-9b4b-44cc-b4c6-d966673eef3f - jinja: 'Translate into English: {{translation["de"]}} - - ||| {{translation["en"]}}' + id: b5866f82-9db7-4b89-9b6f-42168dcf4113 + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-target + - a_good_translation-en-de-target reference: '' - db0bcc05-3732-4931-9559-54c6ec56ec60: !Template + c0bf038f-845b-4e18-be58-cf8501cdd421: !Template answer_choices: null - id: db0bcc05-3732-4931-9559-54c6ec56ec60 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: c0bf038f-845b-4e18-be58-cf8501cdd421 + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target - reference: '' - dc89baca-5066-4560-a98b-5d5fa5bbb0e3: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + c30ea6e2-a77c-4372-8522-bc39a78cede5: !Template answer_choices: null - id: dc89baca-5066-4560-a98b-5d5fa5bbb0e3 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: c30ea6e2-a77c-4372-8522-bc39a78cede5 + jinja: '{{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate_as_de-en-target reference: '' - eb2dd5dd-1e4e-4235-827d-f63d1748015a: !Template + c6e04cfa-dd7d-4623-93f8-179610625f48: !Template answer_choices: null - id: eb2dd5dd-1e4e-4235-827d-f63d1748015a - jinja: 'Translate English into German: {{translation["en"]}} + id: c6e04cfa-dd7d-4623-93f8-179610625f48 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget - reference: '' - f8099b32-c2dd-4b5d-bf8e-2ef9aeedf149: !Template + - gpt-3-de-en-source+target + reference: GPT-3 paper + ceace350-5d39-450b-bb2a-1c3187a19ed8: !Template answer_choices: null - id: f8099b32-c2dd-4b5d-bf8e-2ef9aeedf149 - jinja: 'Translate German into English: {{translation["de"]}} + id: ceace350-5d39-450b-bb2a-1c3187a19ed8 + jinja: 'Translate this from German into English: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -369,3 +341,31 @@ templates: name: !!python/tuple - translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf + d4c41751-65b4-456f-b414-4ed886974c28: !Template + answer_choices: null + id: d4c41751-65b4-456f-b414-4ed886974c28 + jinja: 'German: {{translation["de"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-de-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e7992f8f-a017-40b4-bb15-2ff043a4701b: !Template + answer_choices: null + id: e7992f8f-a017-40b4-bb15-2ff043a4701b + jinja: 'German: {{translation["de"]}} translates into English as: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate_as_de-en-source+target + reference: '' diff --git a/promptsource/templates/wmt18/et-en/templates.yaml b/promptsource/templates/wmt18/et-en/templates.yaml index 172f788ee..ad7155798 100644 --- a/promptsource/templates/wmt18/et-en/templates.yaml +++ b/promptsource/templates/wmt18/et-en/templates.yaml @@ -1,110 +1,109 @@ dataset: wmt18 subset: et-en templates: - 09b66a7c-f6ef-448b-bcfd-0fa8e2ca7d05: !Template + 01c9974b-db75-4833-a1c8-1f43689a5f4b: !Template answer_choices: null - id: 09b66a7c-f6ef-448b-bcfd-0fa8e2ca7d05 - jinja: '{{translation["et"]}} translates into English as: + id: 01c9974b-db75-4833-a1c8-1f43689a5f4b + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Estonian translation is: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_et-en-target + - a_good_translation-en-et-source+target reference: '' - 12e126e6-87e4-4606-998f-274d75c0cdd4: !Template + 0d0e6a01-1673-4296-9a17-d56ec1ed6983: !Template answer_choices: null - id: 12e126e6-87e4-4606-998f-274d75c0cdd4 - jinja: 'English: {{translation["en"]}} translates into Estonian as: - - ||| {{translation["et"]}}' + id: 0d0e6a01-1673-4296-9a17-d56ec1ed6983 + jinja: 'Given the following passage: {{translation["en"]}} , a good Estonian translation + is: ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-et-source+target + - a_good_translation-en-et-target reference: '' - 242d8884-3fab-4304-ae12-75f543979729: !Template + 0f639e5e-15c8-4526-96a5-03deae5754d2: !Template answer_choices: null - id: 242d8884-3fab-4304-ae12-75f543979729 - jinja: 'If the original version says: {{translation["en"]}}; then the Estonian - version should say: + id: 0f639e5e-15c8-4526-96a5-03deae5754d2 + jinja: ' {{translation["en"]}} translates into Estonian as: - ||| {{translation["et"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-et-target + - translate_as_en-et-target reference: '' - 3346e459-02ab-4a67-a0f3-0c4a97a5c7bc: !Template + 1594d8db-e723-4451-8f14-62d933a70f27: !Template answer_choices: null - id: 3346e459-02ab-4a67-a0f3-0c4a97a5c7bc - jinja: '{{translation["en"]}} = Estonian: - - ||| {{translation["et"]}}' + id: 1594d8db-e723-4451-8f14-62d933a70f27 + jinja: 'Given the following source text in Estonian: {{translation["et"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-et-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 367e49e9-b71a-4255-b213-a956c4e53dbc: !Template + - a_good_translation-et-en-source+target + reference: '' + 1a003a8d-1fa6-4d49-b1bf-3510ab894be1: !Template answer_choices: null - id: 367e49e9-b71a-4255-b213-a956c4e53dbc - jinja: 'What is the English translation of : {{translation["et"]}} + id: 1a003a8d-1fa6-4d49-b1bf-3510ab894be1 + jinja: 'What is the Estonian translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-et-en-target - reference: GPT-3 paper - 389fba3f-84b8-4d49-825a-af276178fef4: !Template + - gpt-3-en-et-source+target + reference: GPT 3 paper + 21144c2c-d45d-45e5-a1c1-0f8150fc0dc0: !Template answer_choices: null - id: 389fba3f-84b8-4d49-825a-af276178fef4 - jinja: 'What is the Estonian translation of: {{translation["en"]}} + id: 21144c2c-d45d-45e5-a1c1-0f8150fc0dc0 + jinja: 'Given the following passage: {{translation["et"]}} , a good English translation + is: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-et-target - reference: GPT-3 paper - 4c2d841d-a772-4de2-9eec-d777cde53d0a: !Template + - a_good_translation-et-en-target + reference: '' + 564e1318-7bf6-4e7b-9f9d-9ce31f33717d: !Template answer_choices: null - id: 4c2d841d-a772-4de2-9eec-d777cde53d0a - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Estonian translation is: + id: 564e1318-7bf6-4e7b-9f9d-9ce31f33717d + jinja: 'If the original version says: {{translation["et"]}}; then the English + version should say: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-et-source+target + - version-et-en-target reference: '' - 4d314556-2f70-4987-9537-0011e29f371f: !Template + 652947c9-9fe0-43b0-8645-f9b6f8cbc6b3: !Template answer_choices: null - id: 4d314556-2f70-4987-9537-0011e29f371f - jinja: ' {{translation["en"]}} translates into Estonian as: + id: 652947c9-9fe0-43b0-8645-f9b6f8cbc6b3 + jinja: 'English: {{translation["en"]}} translates into Estonian as: ||| {{translation["et"]}}' metadata: !TemplateMetadata @@ -113,54 +112,54 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-et-target + - translate_as_en-et-source+target reference: '' - 4f053278-5fb1-4162-bf2b-3feb99630ef8: !Template + 674e2912-7071-47a6-9dc2-e29420278f30: !Template answer_choices: null - id: 4f053278-5fb1-4162-bf2b-3feb99630ef8 - jinja: 'Estonian: {{translation["et"]}} = English: + id: 674e2912-7071-47a6-9dc2-e29420278f30 + jinja: 'Translate this into Estonian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-et-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 5287cf78-f2ab-41ec-a611-b516bf1dbe0f: !Template + - translate-en-et-target + reference: '' + 6b0f77c1-ee58-4cec-b3c9-e35f91954ff9: !Template answer_choices: null - id: 5287cf78-f2ab-41ec-a611-b516bf1dbe0f - jinja: 'Translate into Estonian: {{translation["en"]}} + id: 6b0f77c1-ee58-4cec-b3c9-e35f91954ff9 + jinja: 'What is the English translation of the Estonian sentence: {{translation["et"]}} - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-et-target - reference: '' - 64ac7d67-94f2-4aa7-89b1-bc9247224fe1: !Template + - gpt-3-et-en-source+target + reference: GPT-3 paper + 75c32414-407b-4d6e-bcf1-d236457ffb90: !Template answer_choices: null - id: 64ac7d67-94f2-4aa7-89b1-bc9247224fe1 - jinja: 'How do you say {{translation["en"]}} in Estonian? + id: 75c32414-407b-4d6e-bcf1-d236457ffb90 + jinja: 'How do you say {{translation["et"]}} in English? - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-et-target + - how_to_say-et-en-target reference: '' - 65487caf-8d6f-4e36-965c-5bb8f75ab058: !Template + 84b69786-4942-4805-8661-6bc2e710b3af: !Template answer_choices: null - id: 65487caf-8d6f-4e36-965c-5bb8f75ab058 - jinja: 'What is the English translation of the Estonian sentence: {{translation["et"]}} + id: 84b69786-4942-4805-8661-6bc2e710b3af + jinja: 'Translate this from Estonian into English: {{translation["et"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -169,41 +168,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-et-en-source+target - reference: GPT-3 paper - 6591adaf-63ec-4b7f-ab45-f9647eeba93a: !Template + - translate-et-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 8a5e785d-6e6d-498a-9a93-263653893c66: !Template answer_choices: null - id: 6591adaf-63ec-4b7f-ab45-f9647eeba93a - jinja: 'Given the following passage: {{translation["et"]}} , a good English translation - is: + id: 8a5e785d-6e6d-498a-9a93-263653893c66 + jinja: 'If the Estonian version says: {{translation["et"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-et-en-target + - version-et-en-source+target reference: '' - 6831b64b-a425-49df-8f49-447f9dc0bb49: !Template + 8ea02fb0-bb15-43e6-8d0a-d4c0c79a6638: !Template answer_choices: null - id: 6831b64b-a425-49df-8f49-447f9dc0bb49 - jinja: 'English: {{translation["en"]}} = Estonian: + id: 8ea02fb0-bb15-43e6-8d0a-d4c0c79a6638 + jinja: '{{translation["et"]}} translates into English as: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-et-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 6ff1f5f8-4e88-4ed2-a8dd-9064eec947b4: !Template + - translate_as_et-en-target + reference: '' + 97073b9c-fffa-4ba4-9fff-0fb190735af0: !Template answer_choices: null - id: 6ff1f5f8-4e88-4ed2-a8dd-9064eec947b4 - jinja: 'How do you say {{translation["et"]}} in English? + id: 97073b9c-fffa-4ba4-9fff-0fb190735af0 + jinja: 'Estonian: {{translation["et"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -212,27 +211,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-et-en-target + - translate_as_et-en-source+target reference: '' - 727c22ba-ee38-4d39-b7c2-5d3adf65bfb3: !Template + 9f14808e-8058-4949-8d74-44dacd83408e: !Template answer_choices: null - id: 727c22ba-ee38-4d39-b7c2-5d3adf65bfb3 - jinja: 'If the original version says: {{translation["et"]}}; then the English - version should say: + id: 9f14808e-8058-4949-8d74-44dacd83408e + jinja: 'Translate this from English into Estonian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-et-en-target + - translate-en-et-source+starget reference: '' - 7d4c2271-06ba-443c-bdef-dfccc748c723: !Template + aba17c5d-94ee-4073-8ceb-72963fbfa9c2: !Template answer_choices: null - id: 7d4c2271-06ba-443c-bdef-dfccc748c723 - jinja: 'What is the Estonian translation of the English sentence: {{translation["en"]}} + id: aba17c5d-94ee-4073-8ceb-72963fbfa9c2 + jinja: 'What is the Estonian translation of: {{translation["en"]}} ||| {{translation["et"]}}' metadata: !TemplateMetadata @@ -241,131 +239,133 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-et-source+target - reference: GPT 3 paper - 81f0eb3e-7cc7-4bbc-bda1-cd2f2e688bc2: !Template + - gpt-3-en-et-target + reference: GPT-3 paper + b2cfff2d-27a1-4dcb-a544-82258b46825c: !Template answer_choices: null - id: 81f0eb3e-7cc7-4bbc-bda1-cd2f2e688bc2 - jinja: '{{translation["et"]}} = English: + id: b2cfff2d-27a1-4dcb-a544-82258b46825c + jinja: 'What is the English translation of : {{translation["et"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-et-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 88c79995-83fa-4930-a3e1-676e352ea988: !Template + - gpt-3-et-en-target + reference: GPT-3 paper + b3c91f4a-187b-45c7-ab92-16aa02a76547: !Template answer_choices: null - id: 88c79995-83fa-4930-a3e1-676e352ea988 - jinja: 'Given the following passage: {{translation["en"]}} , a good Estonian translation - is: ||| {{translation["et"]}}' + id: b3c91f4a-187b-45c7-ab92-16aa02a76547 + jinja: 'If the English version says: {{translation["en"]}}; then the Estonian + version should say: + + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-et-target + - version-en-et-source+target reference: '' - 8d1af6df-6641-4507-b611-70391e922292: !Template + b4178376-c25f-4de4-8855-02b182f9ed1b: !Template answer_choices: null - id: 8d1af6df-6641-4507-b611-70391e922292 - jinja: 'Translate Estonian into English: {{translation["et"]}} + id: b4178376-c25f-4de4-8855-02b182f9ed1b + jinja: 'How do you say {{translation["en"]}} in Estonian? - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-et-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 8e26f017-66be-4590-81f6-6e9ff144e1f4: !Template + - how_to_say-en-et-target + reference: '' + bb6e89c9-5fd4-43f4-bdeb-ddf8b01b3531: !Template answer_choices: null - id: 8e26f017-66be-4590-81f6-6e9ff144e1f4 - jinja: 'Translate English into Estonian: {{translation["en"]}} + id: bb6e89c9-5fd4-43f4-bdeb-ddf8b01b3531 + jinja: 'Translate this into English: {{translation["et"]}} - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-et-source+starget + - translate-et-en-target reference: '' - 90ab0154-7745-4620-8b45-f655bee6a8e3: !Template + c14f01b7-69eb-44be-be41-4e5698c53de0: !Template answer_choices: null - id: 90ab0154-7745-4620-8b45-f655bee6a8e3 - jinja: 'Estonian: {{translation["et"]}} translates into English as: + id: c14f01b7-69eb-44be-be41-4e5698c53de0 + jinja: '{{translation["en"]}} = Estonian: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_et-en-source+target - reference: '' - a09ab74f-34b1-4a62-b8d1-6fbd29afb941: !Template + - xglm-en-et-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c5bda122-aefe-4ef0-88a8-c3c9d99aecbf: !Template answer_choices: null - id: a09ab74f-34b1-4a62-b8d1-6fbd29afb941 - jinja: 'If the English version says: {{translation["en"]}}; then the Estonian - version should say: + id: c5bda122-aefe-4ef0-88a8-c3c9d99aecbf + jinja: '{{translation["et"]}} = English: - ||| {{translation["et"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-et-source+target - reference: '' - c67279dc-b013-460f-b2dd-b046166faca2: !Template + - xglm-et-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e736c3da-fb88-4969-b60c-e39a9f5d386d: !Template answer_choices: null - id: c67279dc-b013-460f-b2dd-b046166faca2 - jinja: 'Translate into English: {{translation["et"]}} + id: e736c3da-fb88-4969-b60c-e39a9f5d386d + jinja: 'English: {{translation["en"]}} = Estonian: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-et-en-target - reference: '' - e2cc5691-e61d-4bb6-8c61-d7fb67f3e88c: !Template + - xglm-en-et-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + e982d790-b547-412d-beeb-a2870606eb98: !Template answer_choices: null - id: e2cc5691-e61d-4bb6-8c61-d7fb67f3e88c - jinja: 'If the Estonian version says: {{translation["et"]}}; then the English + id: e982d790-b547-412d-beeb-a2870606eb98 + jinja: 'If the original version says: {{translation["en"]}}; then the Estonian version should say: - ||| {{translation["en"]}}' + ||| {{translation["et"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-et-en-source+target + - version-en-et-target reference: '' - f263fce4-4f4a-46e3-983f-3d78dd73bbb9: !Template + fe16099c-1c68-499f-b655-74fa4701d694: !Template answer_choices: null - id: f263fce4-4f4a-46e3-983f-3d78dd73bbb9 - jinja: 'Given the following source text in Estonian: {{translation["et"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: fe16099c-1c68-499f-b655-74fa4701d694 + jinja: 'Estonian: {{translation["et"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-et-en-source+target - reference: '' + - xglm-et-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt18/fi-en/templates.yaml b/promptsource/templates/wmt18/fi-en/templates.yaml index 3b46fe403..81e90f38d 100644 --- a/promptsource/templates/wmt18/fi-en/templates.yaml +++ b/promptsource/templates/wmt18/fi-en/templates.yaml @@ -1,37 +1,37 @@ dataset: wmt18 subset: fi-en templates: - 024c1b4c-8c16-48bf-aa67-77b20793023b: !Template + 1082386b-8a41-452b-b904-c22e97edd78f: !Template answer_choices: null - id: 024c1b4c-8c16-48bf-aa67-77b20793023b - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: 1082386b-8a41-452b-b904-c22e97edd78f + jinja: 'Translate this into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target - reference: GPT-3 paper - 04b5ce95-a44f-4088-8236-f4620f34f680: !Template + - translate-fi-en-target + reference: '' + 1b1d85d1-32b2-44a2-b3ec-2fea89d2c02e: !Template answer_choices: null - id: 04b5ce95-a44f-4088-8236-f4620f34f680 - jinja: 'English: {{translation["en"]}} = Finnish: + id: 1b1d85d1-32b2-44a2-b3ec-2fea89d2c02e + jinja: '{{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 0b1028e7-82bd-4017-9cf7-9709338aec4e: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 21ee5ae4-9c75-4298-83be-b038d5153501: !Template answer_choices: null - id: 0b1028e7-82bd-4017-9cf7-9709338aec4e + id: 21ee5ae4-9c75-4298-83be-b038d5153501 jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -42,10 +42,10 @@ templates: name: !!python/tuple - a_good_translation-en-fi-target reference: '' - 0fd8d2ba-6751-4f27-a52f-e17aaa390a88: !Template + 24edf4be-dbc0-460e-b125-8735ff8f35ca: !Template answer_choices: null - id: 0fd8d2ba-6751-4f27-a52f-e17aaa390a88 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 24edf4be-dbc0-460e-b125-8735ff8f35ca + jinja: 'English: {{translation["en"]}} = Finnish: ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -54,39 +54,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target - reference: '' - 19b94af0-cbb8-462c-8d3e-e0f9325931be: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 3c191620-56a7-4df1-9ed1-0eca5d0e8616: !Template answer_choices: null - id: 19b94af0-cbb8-462c-8d3e-e0f9325931be - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 3c191620-56a7-4df1-9ed1-0eca5d0e8616 + jinja: 'Translate this from Finnish into English: {{translation["fi"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target - reference: '' - 2d1ae41e-155c-4bc7-8795-0954d0d64136: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 4688e425-9c23-4ebf-a612-e45cf1c82c7e: !Template answer_choices: null - id: 2d1ae41e-155c-4bc7-8795-0954d0d64136 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: 4688e425-9c23-4ebf-a612-e45cf1c82c7e + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target - reference: '' - 39b1cc28-71b1-4f0b-9f4e-49655855dfd2: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 4cccb82d-2d62-4298-ae40-f61ed2beeb37: !Template answer_choices: null - id: 39b1cc28-71b1-4f0b-9f4e-49655855dfd2 - jinja: 'How do you say {{translation["fi"]}} in English? + id: 4cccb82d-2d62-4298-ae40-f61ed2beeb37 + jinja: 'Finnish: {{translation["fi"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -95,241 +96,241 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target - reference: '' - 3ca1c531-f22a-4497-95fe-83c85d6bd0a9: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 51029e7c-4d7c-41bf-8679-a1c931eb5263: !Template answer_choices: null - id: 3ca1c531-f22a-4497-95fe-83c85d6bd0a9 - jinja: 'What is the English translation of : {{translation["fi"]}} + id: 51029e7c-4d7c-41bf-8679-a1c931eb5263 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: GPT-3 paper - 494f82a7-2d20-4786-b1b6-08d985b8f1a5: !Template + - version-fi-en-source+target + reference: '' + 52bb4805-756f-4c74-8573-fe62001f708e: !Template answer_choices: null - id: 494f82a7-2d20-4786-b1b6-08d985b8f1a5 - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: - - ||| {{translation["fi"]}}' + id: 52bb4805-756f-4c74-8573-fe62001f708e + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target + - a_good_translation-fi-en-source+target reference: '' - 4f12e1fb-7363-4357-a92a-8f1f8e5f5f9c: !Template + 566b077a-9e88-480b-8c2c-962b07eb8dbd: !Template answer_choices: null - id: 4f12e1fb-7363-4357-a92a-8f1f8e5f5f9c - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: 566b077a-9e88-480b-8c2c-962b07eb8dbd + jinja: 'Translate this from English into Finnish: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target + - translate-en-fi-source+starget reference: '' - 5386e7c0-40b2-4c0e-b85c-a2d4b7305937: !Template + 58646943-00c3-4b1a-b539-abda1af73620: !Template answer_choices: null - id: 5386e7c0-40b2-4c0e-b85c-a2d4b7305937 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 58646943-00c3-4b1a-b539-abda1af73620 + jinja: 'What is the Finnish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target - reference: '' - 6b8fc7a5-a109-4fcd-9e36-a52851cacd7d: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + 6cbadfd5-e2d1-4ee4-8f78-dd028f1b4fc2: !Template answer_choices: null - id: 6b8fc7a5-a109-4fcd-9e36-a52851cacd7d - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 6cbadfd5-e2d1-4ee4-8f78-dd028f1b4fc2 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - a_good_translation-en-fi-source+target reference: '' - 6bef34c4-2e32-4f0c-b916-dc4c82626023: !Template + 739bde08-a704-4204-9b72-6a4981f83991: !Template answer_choices: null - id: 6bef34c4-2e32-4f0c-b916-dc4c82626023 - jinja: '{{translation["fi"]}} translates into English as: + id: 739bde08-a704-4204-9b72-6a4981f83991 + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target - reference: '' - 74535955-e9bf-4450-9f1c-197a90c76f62: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + 8e8d032a-4f22-4cdf-8861-3a6fc79f148f: !Template answer_choices: null - id: 74535955-e9bf-4450-9f1c-197a90c76f62 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 8e8d032a-4f22-4cdf-8861-3a6fc79f148f + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 7efa59e0-dd55-485e-b70c-8789f97cd7bc: !Template + - version-en-fi-target + reference: '' + 8f85f8aa-c698-4219-871a-35480c4c3311: !Template answer_choices: null - id: 7efa59e0-dd55-485e-b70c-8789f97cd7bc - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 8f85f8aa-c698-4219-871a-35480c4c3311 + jinja: 'English: {{translation["en"]}} translates into Finnish as: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - translate_as_en-fi-source+target reference: '' - 896cd18d-d469-414e-959e-5925d12461ed: !Template + 930addb7-0d80-4ddd-8f21-729693d8cb37: !Template answer_choices: null - id: 896cd18d-d469-414e-959e-5925d12461ed - jinja: 'English: {{translation["en"]}} translates into Finnish as: + id: 930addb7-0d80-4ddd-8f21-729693d8cb37 + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - a_good_translation-fi-en-target reference: '' - 9455177d-c8d8-4106-bf52-239aef915196: !Template + ac5207f1-7499-45d2-a570-9c6f4e751810: !Template answer_choices: null - id: 9455177d-c8d8-4106-bf52-239aef915196 - jinja: 'Finnish: {{translation["fi"]}} = English: + id: ac5207f1-7499-45d2-a570-9c6f4e751810 + jinja: 'Translate this into Finnish: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9488651a-e1ef-42d5-ae82-b5e0b2c8a98c: !Template + - translate-en-fi-target + reference: '' + ae27d437-3c89-44d4-a268-586c5835d299: !Template answer_choices: null - id: 9488651a-e1ef-42d5-ae82-b5e0b2c8a98c - jinja: '{{translation["en"]}} = Finnish: + id: ae27d437-3c89-44d4-a268-586c5835d299 + jinja: '{{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9c3f1a73-4b97-4e9a-b0c8-0657109974d9: !Template + - translate_as_fi-en-target + reference: '' + bcf156c8-8d81-46ea-b511-432537cbcefa: !Template answer_choices: null - id: 9c3f1a73-4b97-4e9a-b0c8-0657109974d9 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: bcf156c8-8d81-46ea-b511-432537cbcefa + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target + - version-en-fi-source+target reference: '' - b54d9dfe-bdd3-4c66-b698-d1fef339b856: !Template + c81d082d-c493-44db-b72a-d82616cbbe21: !Template answer_choices: null - id: b54d9dfe-bdd3-4c66-b698-d1fef339b856 - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} + id: c81d082d-c493-44db-b72a-d82616cbbe21 + jinja: 'Finnish: {{translation["fi"]}} translates into English as: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-source+target - reference: GPT 3 paper - bd308723-64e4-4765-951f-6e10317a43fd: !Template + - translate_as_fi-en-source+target + reference: '' + d8d03f61-9811-4af2-8ac2-c11356c30ac5: !Template answer_choices: null - id: bd308723-64e4-4765-951f-6e10317a43fd - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: d8d03f61-9811-4af2-8ac2-c11356c30ac5 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-source+target - reference: GPT-3 paper - c28c0e87-cba7-4374-bc79-c0bf36de7a23: !Template + - xglm-en-fi-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + df798200-71a8-4c46-8414-57b075a34825: !Template answer_choices: null - id: c28c0e87-cba7-4374-bc79-c0bf36de7a23 - jinja: 'Translate into English: {{translation["fi"]}} + id: df798200-71a8-4c46-8414-57b075a34825 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target + - version-fi-en-target reference: '' - cb5a91d5-c7ea-464e-8eb2-d5f029d176c2: !Template + e44885b9-8c9c-4c5e-942d-851c9956cfae: !Template answer_choices: null - id: cb5a91d5-c7ea-464e-8eb2-d5f029d176c2 - jinja: '{{translation["fi"]}} = English: + id: e44885b9-8c9c-4c5e-942d-851c9956cfae + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - da6870c8-f33e-4809-a446-f72a0f376d40: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + edc5f88a-155e-4650-96cc-c10c9ba7b7cf: !Template answer_choices: null - id: da6870c8-f33e-4809-a446-f72a0f376d40 - jinja: ' {{translation["en"]}} translates into Finnish as: + id: edc5f88a-155e-4650-96cc-c10c9ba7b7cf + jinja: 'How do you say {{translation["en"]}} in Finnish? ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -338,34 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - how_to_say-en-fi-target reference: '' - f715353e-4152-450b-8e6e-baf34a7b0672: !Template + f657dc01-987f-4aa6-85a5-482f164750d9: !Template answer_choices: null - id: f715353e-4152-450b-8e6e-baf34a7b0672 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: f657dc01-987f-4aa6-85a5-482f164750d9 + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target + - translate_as_en-fi-target reference: '' - fe8a576e-64f0-497a-a831-290cc1b693fd: !Template + fa40e79e-ba97-4a88-a39d-ee4e4f0aaba3: !Template answer_choices: null - id: fe8a576e-64f0-497a-a831-290cc1b693fd - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: fa40e79e-ba97-4a88-a39d-ee4e4f0aaba3 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-source+starget + - how_to_say-fi-en-target reference: '' diff --git a/promptsource/templates/wmt18/ru-en/templates.yaml b/promptsource/templates/wmt18/ru-en/templates.yaml index 9cd03c8b0..3b43dc1bf 100644 --- a/promptsource/templates/wmt18/ru-en/templates.yaml +++ b/promptsource/templates/wmt18/ru-en/templates.yaml @@ -1,54 +1,52 @@ dataset: wmt18 subset: ru-en templates: - 01d74beb-afae-44b5-b29e-02041882f8ec: !Template + 077d0095-f11f-4084-849f-02ce3285fe58: !Template answer_choices: null - id: 01d74beb-afae-44b5-b29e-02041882f8ec - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: + id: 077d0095-f11f-4084-849f-02ce3285fe58 + jinja: 'Russian: {{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target - reference: '' - 06c3b21a-d5da-443d-950a-d826a360e209: !Template + - xglm-ru-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 0f0cb8d1-0967-4b39-98d9-90f7f83021c9: !Template answer_choices: null - id: 06c3b21a-d5da-443d-950a-d826a360e209 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 0f0cb8d1-0967-4b39-98d9-90f7f83021c9 + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-source+target - reference: '' - 0a6e3220-63d8-452c-8569-5fa60368ddc9: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 1128f6e3-4a33-4793-8ac8-c167a198149d: !Template answer_choices: null - id: 0a6e3220-63d8-452c-8569-5fa60368ddc9 - jinja: 'If the original version says: {{translation["ru"]}}; then the English - version should say: + id: 1128f6e3-4a33-4793-8ac8-c167a198149d + jinja: 'Translate this from English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-target + - translate-en-ru-source+starget reference: '' - 14dc3a24-b45a-4e7c-b039-160162fd559d: !Template + 1a45a068-7b2d-43c7-a998-63ee7f6f8e19: !Template answer_choices: null - id: 14dc3a24-b45a-4e7c-b039-160162fd559d - jinja: ' {{translation["en"]}} translates into Russian as: + id: 1a45a068-7b2d-43c7-a998-63ee7f6f8e19 + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -57,166 +55,169 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - translate_as_en-ru-source+target reference: '' - 1e0006f8-30fa-49ea-bbbf-5e0c4cb1ef01: !Template + 2c67569d-bca3-4552-94fc-a963ab379d53: !Template answer_choices: null - id: 1e0006f8-30fa-49ea-bbbf-5e0c4cb1ef01 - jinja: 'Translate into English: {{translation["ru"]}} + id: 2c67569d-bca3-4552-94fc-a963ab379d53 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target + - a_good_translation-en-ru-source+target reference: '' - 272d9f29-4228-4b3f-b996-1c9ca199f4ad: !Template + 35bab45f-70b1-42b4-aaf5-f113228e60fb: !Template answer_choices: null - id: 272d9f29-4228-4b3f-b996-1c9ca199f4ad - jinja: '{{translation["en"]}} = Russian: + id: 35bab45f-70b1-42b4-aaf5-f113228e60fb + jinja: 'Russian: {{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 4224a9fa-fcf7-4613-a4d7-f14454ad367d: !Template + - translate_as_ru-en-source+target + reference: '' + 428e554a-22a5-437b-a5e8-946749978e27: !Template answer_choices: null - id: 4224a9fa-fcf7-4613-a4d7-f14454ad367d - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 428e554a-22a5-437b-a5e8-946749978e27 + jinja: 'What is the Russian translation of: {{translation["en"]}} - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target - reference: '' - 4395a1cc-639b-481b-9ffc-2483b1fd0954: !Template + - gpt-3-en-ru-target + reference: GPT-3 paper + 50ad6cb7-8b62-4144-bce0-8a7464f8f755: !Template answer_choices: null - id: 4395a1cc-639b-481b-9ffc-2483b1fd0954 - jinja: 'English: {{translation["en"]}} = Russian: - - ||| {{translation["ru"]}}' + id: 50ad6cb7-8b62-4144-bce0-8a7464f8f755 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 45cf6192-fee9-4e82-a4b4-51718ba2b9df: !Template + - a_good_translation-en-ru-target + reference: '' + 6347c761-2ac5-42e2-8c82-6081417d1d79: !Template answer_choices: null - id: 45cf6192-fee9-4e82-a4b4-51718ba2b9df - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 6347c761-2ac5-42e2-8c82-6081417d1d79 + jinja: 'English: {{translation["en"]}} = Russian: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-source+target - reference: GPT-3 paper - 5909ecf7-85ea-42b8-b441-db560c81231f: !Template + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 65337a40-e02a-46ab-9c95-3bda83467260: !Template answer_choices: null - id: 5909ecf7-85ea-42b8-b441-db560c81231f - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 65337a40-e02a-46ab-9c95-3bda83467260 + jinja: 'Translate this from Russian into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target - reference: '' - 5b17adea-cfca-444f-8dec-aecc27168cad: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 6c3fad5f-bae7-473e-a8d8-0e5da0803bac: !Template answer_choices: null - id: 5b17adea-cfca-444f-8dec-aecc27168cad - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 6c3fad5f-bae7-473e-a8d8-0e5da0803bac + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: + + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target + - version-en-ru-source+target reference: '' - 619a1ee7-d2f3-46cc-9a88-fae912ab7d12: !Template + 6ead070d-cb04-4236-84f5-120f40c80e3a: !Template answer_choices: null - id: 619a1ee7-d2f3-46cc-9a88-fae912ab7d12 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 6ead070d-cb04-4236-84f5-120f40c80e3a + jinja: 'What is the English translation of : {{translation["ru"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target - reference: '' - 634ce49c-d8b8-4715-9dd1-1c0b660ab7ba: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 838a5379-088e-4d06-b065-d45d70c3d9b5: !Template answer_choices: null - id: 634ce49c-d8b8-4715-9dd1-1c0b660ab7ba - jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation - is: + id: 838a5379-088e-4d06-b065-d45d70c3d9b5 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-target + - version-en-ru-target reference: '' - 6b990f93-754b-48bc-a1fd-afff9262fa2b: !Template + 8cd34877-7933-4b8c-922e-8e42f6efd9d8: !Template answer_choices: null - id: 6b990f93-754b-48bc-a1fd-afff9262fa2b - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 8cd34877-7933-4b8c-922e-8e42f6efd9d8 + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - 6d0b2f15-f224-4c7f-8073-63de87134d6e: !Template + - translate_as_en-ru-target + reference: '' + 96d0086c-c7be-4513-8448-2ae104d8829e: !Template answer_choices: null - id: 6d0b2f15-f224-4c7f-8073-63de87134d6e - jinja: '{{translation["ru"]}} = English: + id: 96d0086c-c7be-4513-8448-2ae104d8829e + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 95e1218e-e453-4c4f-8bd1-d0eae3c14d12: !Template + - version-ru-en-source+target + reference: '' + 996c66b5-57db-4fdd-b5fb-c4517d3ac8f3: !Template answer_choices: null - id: 95e1218e-e453-4c4f-8bd1-d0eae3c14d12 - jinja: '{{translation["ru"]}} translates into English as: + id: 996c66b5-57db-4fdd-b5fb-c4517d3ac8f3 + jinja: 'How do you say {{translation["ru"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -225,147 +226,146 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - how_to_say-ru-en-target reference: '' - 9723af4f-23fd-451a-8c0e-082bc57dc139: !Template + 9f9b91df-4a77-4e29-a82a-002ed8d83bf7: !Template answer_choices: null - id: 9723af4f-23fd-451a-8c0e-082bc57dc139 - jinja: 'What is the Russian translation of: {{translation["en"]}} + id: 9f9b91df-4a77-4e29-a82a-002ed8d83bf7 + jinja: 'If the original version says: {{translation["ru"]}}; then the English + version should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-target - reference: GPT-3 paper - 9975a351-b686-4616-9e02-181d4388f283: !Template + - version-ru-en-target + reference: '' + a985bdd7-e165-486a-880d-c68c32acb42c: !Template answer_choices: null - id: 9975a351-b686-4616-9e02-181d4388f283 - jinja: 'Translate into Russian: {{translation["en"]}} + id: a985bdd7-e165-486a-880d-c68c32acb42c + jinja: 'Translate this into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target + - translate-ru-en-target reference: '' - 9ef1e600-e7e0-4ad4-8ba0-df4eb5abacf1: !Template + ac5b39bf-8acc-42ed-bc0d-6723dcd8761e: !Template answer_choices: null - id: 9ef1e600-e7e0-4ad4-8ba0-df4eb5abacf1 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: ac5b39bf-8acc-42ed-bc0d-6723dcd8761e + jinja: '{{translation["ru"]}} translates into English as: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - translate_as_ru-en-target reference: '' - a6e12617-5232-485d-a1a4-0a897dd8f826: !Template + b3ea9108-afd4-4ade-a80f-4f9c514910ec: !Template answer_choices: null - id: a6e12617-5232-485d-a1a4-0a897dd8f826 - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: b3ea9108-afd4-4ade-a80f-4f9c514910ec + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target - reference: '' - cd27bbef-a7ac-4db0-95a5-e2649877fcac: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + b412f718-09ca-4003-9e6b-0abb2e1701c0: !Template answer_choices: null - id: cd27bbef-a7ac-4db0-95a5-e2649877fcac - jinja: 'Translate English into Russian: {{translation["en"]}} + id: b412f718-09ca-4003-9e6b-0abb2e1701c0 + jinja: '{{translation["ru"]}} = English: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-source+starget - reference: '' - d4f5b5a0-c050-4eea-bfd4-3bb14fb6eb5b: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + b4484ec0-f128-44c4-a639-49e26645f19f: !Template answer_choices: null - id: d4f5b5a0-c050-4eea-bfd4-3bb14fb6eb5b - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: b4484ec0-f128-44c4-a639-49e26645f19f + jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e2baed91-95c3-40fa-b027-fc1d0344a932: !Template + - a_good_translation-ru-en-target + reference: '' + bd2c6a7d-09f2-4082-ba20-e53bc7fd7a89: !Template answer_choices: null - id: e2baed91-95c3-40fa-b027-fc1d0344a932 - jinja: 'How do you say {{translation["ru"]}} in English? + id: bd2c6a7d-09f2-4082-ba20-e53bc7fd7a89 + jinja: 'Translate this into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate-en-ru-target reference: '' - e5810f14-cd42-49dd-9542-51034230f4ff: !Template + c43b019f-3c4a-43ee-a3a1-6346d0beab8c: !Template answer_choices: null - id: e5810f14-cd42-49dd-9542-51034230f4ff - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: c43b019f-3c4a-43ee-a3a1-6346d0beab8c + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - e6bffe73-347f-4940-baa3-609f3a7a0528: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + c9a66968-e695-4b22-b83a-ddc89c6220de: !Template answer_choices: null - id: e6bffe73-347f-4940-baa3-609f3a7a0528 - jinja: 'Russian: {{translation["ru"]}} = English: + id: c9a66968-e695-4b22-b83a-ddc89c6220de + jinja: 'How do you say {{translation["en"]}} in Russian? - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - feff3a10-1258-48a7-91d2-d686417d6c24: !Template + - how_to_say-en-ru-target + reference: '' + fba7ed4e-8cf6-4da3-b27a-43400a514ed8: !Template answer_choices: null - id: feff3a10-1258-48a7-91d2-d686417d6c24 - jinja: 'Russian: {{translation["ru"]}} translates into English as: - - ||| {{translation["en"]}}' + id: fba7ed4e-8cf6-4da3-b27a-43400a514ed8 + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - a_good_translation-ru-en-source+target reference: '' diff --git a/promptsource/templates/wmt18/tr-en/templates.yaml b/promptsource/templates/wmt18/tr-en/templates.yaml index 991653512..81cc050ec 100644 --- a/promptsource/templates/wmt18/tr-en/templates.yaml +++ b/promptsource/templates/wmt18/tr-en/templates.yaml @@ -1,124 +1,121 @@ dataset: wmt18 subset: tr-en templates: - 016f1160-28ce-4177-ab95-f3c28d2a06c1: !Template + 06dc2e5d-c236-4a78-9476-85d7e9e26e75: !Template answer_choices: null - id: 016f1160-28ce-4177-ab95-f3c28d2a06c1 - jinja: 'What is the English translation of : {{translation["tr"]}} - - ||| {{translation["en"]}}' + id: 06dc2e5d-c236-4a78-9476-85d7e9e26e75 + jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-target - reference: GPT-3 paper - 06300fbd-1570-404a-abfb-5ca26710a1d0: !Template + - a_good_translation-tr-en-source+target + reference: '' + 251bf17b-2604-45d3-a875-8c08227de84a: !Template answer_choices: null - id: 06300fbd-1570-404a-abfb-5ca26710a1d0 - jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} + id: 251bf17b-2604-45d3-a875-8c08227de84a + jinja: '{{translation["tr"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-tr-en-source+target - reference: GPT-3 paper - 162fba92-4660-4293-8a92-8bdcbc90d739: !Template + - translate_as_tr-en-target + reference: '' + 35036ac2-06b6-4899-89b2-c96436ab3c4e: !Template answer_choices: null - id: 162fba92-4660-4293-8a92-8bdcbc90d739 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Turkish translation is: + id: 35036ac2-06b6-4899-89b2-c96436ab3c4e + jinja: '{{translation["tr"]}} = English: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-source+target - reference: '' - 17064a42-05cd-497f-8387-64bf4083a6f2: !Template + - xglm-tr-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 3588d4c9-4a11-41cb-b548-37d04f0f9b66: !Template answer_choices: null - id: 17064a42-05cd-497f-8387-64bf4083a6f2 - jinja: 'If the original version says: {{translation["en"]}}; then the Turkish - version should say: + id: 3588d4c9-4a11-41cb-b548-37d04f0f9b66 + jinja: 'Translate this into Turkish: {{translation["en"]}} - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-target + - translate-en-tr-target reference: '' - 22ef815c-051c-460a-96cb-f281eae08d06: !Template + 3afad62b-2cab-4c06-965a-195588c5ef70: !Template answer_choices: null - id: 22ef815c-051c-460a-96cb-f281eae08d06 - jinja: 'Translate English into Turkish: {{translation["en"]}} + id: 3afad62b-2cab-4c06-965a-195588c5ef70 + jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version + should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-source+starget + - version-en-tr-source+target reference: '' - 274ee105-b03a-4fad-88da-fe6df67ecae6: !Template + 427c296e-ff3f-4d9c-8ed1-cc1900ad4d5f: !Template answer_choices: null - id: 274ee105-b03a-4fad-88da-fe6df67ecae6 - jinja: 'How do you say {{translation["tr"]}} in English? + id: 427c296e-ff3f-4d9c-8ed1-cc1900ad4d5f + jinja: ' {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-tr-en-target + - translate_as_en-tr-target reference: '' - 365d3c5a-abbf-486d-81d1-07974492fbd5: !Template + 44842d8e-fb52-43cd-a042-2a5145e28b9a: !Template answer_choices: null - id: 365d3c5a-abbf-486d-81d1-07974492fbd5 - jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} - - ||| {{translation["tr"]}}' + id: 44842d8e-fb52-43cd-a042-2a5145e28b9a + jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation + is: ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-source+target - reference: GPT 3 paper - 3f36f729-8f09-4750-b935-0b68b3baf3b6: !Template + - a_good_translation-en-tr-target + reference: '' + 5b01d692-c545-46d6-adce-5a889af5d71e: !Template answer_choices: null - id: 3f36f729-8f09-4750-b935-0b68b3baf3b6 - jinja: 'Translate Turkish into English: {{translation["tr"]}} + id: 5b01d692-c545-46d6-adce-5a889af5d71e + jinja: 'What is the Turkish translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 4081db0d-365d-473d-bf0e-de8cb267a8d1: !Template + - gpt-3-en-tr-target + reference: GPT-3 paper + 5df2bcb3-45d7-47c1-ac4c-24ee23041c21: !Template answer_choices: null - id: 4081db0d-365d-473d-bf0e-de8cb267a8d1 - jinja: '{{translation["tr"]}} = English: + id: 5df2bcb3-45d7-47c1-ac4c-24ee23041c21 + jinja: 'How do you say {{translation["tr"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -127,26 +124,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 665c537e-e5f2-454e-ad32-fe81c5974905: !Template + - how_to_say-tr-en-target + reference: '' + 6a79649a-46aa-4048-9178-766117a9faa5: !Template answer_choices: null - id: 665c537e-e5f2-454e-ad32-fe81c5974905 - jinja: 'English: {{translation["en"]}} = Turkish: + id: 6a79649a-46aa-4048-9178-766117a9faa5 + jinja: 'Translate this from Turkish into English: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 6c8531d9-2003-4d45-8d67-59c67b71d7fd: !Template + - translate-tr-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 700aa770-08cb-4fc5-b539-19cf26d8309b: !Template answer_choices: null - id: 6c8531d9-2003-4d45-8d67-59c67b71d7fd - jinja: ' {{translation["en"]}} translates into Turkish as: + id: 700aa770-08cb-4fc5-b539-19cf26d8309b + jinja: 'How do you say {{translation["en"]}} in Turkish? ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -155,27 +152,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-target + - how_to_say-en-tr-target reference: '' - 6f78297a-47ff-4027-b3b0-d77d8663a1a9: !Template + 71825d8e-79d6-4ca4-82c2-6647af66432c: !Template answer_choices: null - id: 6f78297a-47ff-4027-b3b0-d77d8663a1a9 - jinja: 'Translate into Turkish: {{translation["en"]}} + id: 71825d8e-79d6-4ca4-82c2-6647af66432c + jinja: 'If the original version says: {{translation["en"]}}; then the Turkish + version should say: - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-tr-target + - version-en-tr-target reference: '' - 71e3aef0-0719-4709-ae26-20740cb11360: !Template + 7405c125-29ff-440a-bdc0-d1d8ffaff04b: !Template answer_choices: null - id: 71e3aef0-0719-4709-ae26-20740cb11360 - jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation - is: + id: 7405c125-29ff-440a-bdc0-d1d8ffaff04b + jinja: 'Turkish: {{translation["tr"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -184,109 +181,112 @@ templates: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-target + - translate_as_tr-en-source+target reference: '' - 801f0c40-fb74-4bf5-b21d-58855c8388ea: !Template + 76b7109c-6307-468c-8d14-23ac760fd22c: !Template answer_choices: null - id: 801f0c40-fb74-4bf5-b21d-58855c8388ea - jinja: 'Given the following source text in Turkish: {{translation["tr"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 76b7109c-6307-468c-8d14-23ac760fd22c + jinja: 'Turkish: {{translation["tr"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-tr-en-source+target - reference: '' - b08a3b75-f728-4255-98ed-2c29474d568c: !Template + - xglm-tr-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 85e15d3a-462a-4de9-a854-270a580d3909: !Template answer_choices: null - id: b08a3b75-f728-4255-98ed-2c29474d568c - jinja: 'Given the following passage: {{translation["en"]}} , a good Turkish translation - is: ||| {{translation["tr"]}}' + id: 85e15d3a-462a-4de9-a854-270a580d3909 + jinja: 'Given the following passage: {{translation["tr"]}} , a good English translation + is: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-tr-target + - a_good_translation-tr-en-target reference: '' - b283beaf-cdf2-4939-b91a-cd6c7fafbdfd: !Template + 9b08536a-517a-49b2-a446-636f0c2b608c: !Template answer_choices: null - id: b283beaf-cdf2-4939-b91a-cd6c7fafbdfd - jinja: 'English: {{translation["en"]}} translates into Turkish as: + id: 9b08536a-517a-49b2-a446-636f0c2b608c + jinja: 'What is the English translation of the Turkish sentence: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-tr-source+target - reference: '' - c2bd250c-1c4e-47ea-99fa-7f5bd83ec2c7: !Template + - gpt-3-tr-en-source+target + reference: GPT-3 paper + 9f577eb5-1d96-492b-843e-ea656dd56964: !Template answer_choices: null - id: c2bd250c-1c4e-47ea-99fa-7f5bd83ec2c7 - jinja: '{{translation["tr"]}} translates into English as: + id: 9f577eb5-1d96-492b-843e-ea656dd56964 + jinja: 'What is the English translation of : {{translation["tr"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-target - reference: '' - c8d30004-2ad5-45ac-9891-4b35fecb0193: !Template + - gpt-3-tr-en-target + reference: GPT-3 paper + a7118387-ce27-4852-ae7a-634e07e493a7: !Template answer_choices: null - id: c8d30004-2ad5-45ac-9891-4b35fecb0193 - jinja: 'If the English version says: {{translation["en"]}}; then the Turkish version - should say: + id: a7118387-ce27-4852-ae7a-634e07e493a7 + jinja: 'Translate this into English: {{translation["tr"]}} - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-tr-source+target + - translate-tr-en-target reference: '' - c8eb00dd-3720-4cf0-a93b-742f087b9a8e: !Template + ae1b2105-85a8-417e-8edf-87170aab6d27: !Template answer_choices: null - id: c8eb00dd-3720-4cf0-a93b-742f087b9a8e - jinja: 'Translate into English: {{translation["tr"]}} + id: ae1b2105-85a8-417e-8edf-87170aab6d27 + jinja: 'If the original version says: {{translation["tr"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-tr-en-target + - version-tr-en-target reference: '' - e2aa69d8-b11a-43df-900e-ad64e434ea9e: !Template + b6fe589c-bfea-433e-ab96-5b921693ebfa: !Template answer_choices: null - id: e2aa69d8-b11a-43df-900e-ad64e434ea9e - jinja: 'How do you say {{translation["en"]}} in Turkish? + id: b6fe589c-bfea-433e-ab96-5b921693ebfa + jinja: 'Translate this from English into Turkish: {{translation["en"]}} - ||| {{translation["tr"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-tr-target + - translate-en-tr-source+starget reference: '' - e487cf87-2e5f-44b2-a026-2978f4d27f8a: !Template + ca255baf-3552-40bf-b2c8-4bd3957b86a3: !Template answer_choices: null - id: e487cf87-2e5f-44b2-a026-2978f4d27f8a - jinja: 'What is the Turkish translation of: {{translation["en"]}} + id: ca255baf-3552-40bf-b2c8-4bd3957b86a3 + jinja: 'What is the Turkish translation of the English sentence: {{translation["en"]}} ||| {{translation["tr"]}}' metadata: !TemplateMetadata @@ -295,77 +295,77 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-tr-target - reference: GPT-3 paper - e7d9f4d5-5687-4337-b0fb-fea27b4581e3: !Template + - gpt-3-en-tr-source+target + reference: GPT 3 paper + cdc4d883-eb4e-40bc-bd56-3043d3ca6303: !Template answer_choices: null - id: e7d9f4d5-5687-4337-b0fb-fea27b4581e3 - jinja: 'If the original version says: {{translation["tr"]}}; then the English - version should say: + id: cdc4d883-eb4e-40bc-bd56-3043d3ca6303 + jinja: '{{translation["en"]}} = Turkish: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-target - reference: '' - e9fed2be-f4d5-4437-a080-68db1c640f34: !Template + - xglm-en-tr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + dbac9603-dc7c-4d1f-827f-e5b70f3bd023: !Template answer_choices: null - id: e9fed2be-f4d5-4437-a080-68db1c640f34 - jinja: 'Turkish: {{translation["tr"]}} translates into English as: + id: dbac9603-dc7c-4d1f-827f-e5b70f3bd023 + jinja: 'English: {{translation["en"]}} translates into Turkish as: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_tr-en-source+target + - translate_as_en-tr-source+target reference: '' - ed78f056-9059-4a95-821e-baef0831218a: !Template + dbe3c495-165f-4d7d-af8f-97c2de1cba94: !Template answer_choices: null - id: ed78f056-9059-4a95-821e-baef0831218a - jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version - should say: + id: dbe3c495-165f-4d7d-af8f-97c2de1cba94 + jinja: 'English: {{translation["en"]}} = Turkish: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-tr-en-source+target - reference: '' - f5942e98-b1db-4cd9-895d-3270aa9e15ce: !Template + - xglm-en-tr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + e80e51a0-188c-4bfb-8d3e-00048b6ddaa6: !Template answer_choices: null - id: f5942e98-b1db-4cd9-895d-3270aa9e15ce - jinja: '{{translation["en"]}} = Turkish: + id: e80e51a0-188c-4bfb-8d3e-00048b6ddaa6 + jinja: 'If the Turkish version says: {{translation["tr"]}}; then the English version + should say: - ||| {{translation["tr"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-tr-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f99a028e-61d1-4d1b-a75a-388d21cc253b: !Template + - version-tr-en-source+target + reference: '' + f7ac15b6-0d18-4d3c-b9f9-4ae893f8ae2c: !Template answer_choices: null - id: f99a028e-61d1-4d1b-a75a-388d21cc253b - jinja: 'Turkish: {{translation["tr"]}} = English: + id: f7ac15b6-0d18-4d3c-b9f9-4ae893f8ae2c + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Turkish translation is: - ||| {{translation["en"]}}' + ||| {{translation["tr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-tr-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 + - a_good_translation-en-tr-source+target + reference: '' diff --git a/promptsource/templates/wmt18/zh-en/templates.yaml b/promptsource/templates/wmt18/zh-en/templates.yaml index c2e07dc60..8a4de53c9 100644 --- a/promptsource/templates/wmt18/zh-en/templates.yaml +++ b/promptsource/templates/wmt18/zh-en/templates.yaml @@ -1,108 +1,109 @@ dataset: wmt18 subset: zh-en templates: - 0217111b-6e6f-4b66-a014-e904a87d9ba0: !Template + 326a4521-9a99-45af-9919-94ef67d9c6a5: !Template answer_choices: null - id: 0217111b-6e6f-4b66-a014-e904a87d9ba0 - jinja: '{{translation["zh"]}} translates into English as: + id: 326a4521-9a99-45af-9919-94ef67d9c6a5 + jinja: 'English: {{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target - reference: '' - 132b17ab-24be-4311-b78c-a9120a66ad30: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 3c40fb69-5e39-4e86-8041-7df857ca8eff: !Template answer_choices: null - id: 132b17ab-24be-4311-b78c-a9120a66ad30 - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + id: 3c40fb69-5e39-4e86-8041-7df857ca8eff + jinja: 'If the original version says: {{translation["zh"]}}; then the English version should say: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target + - version-zh-en-target reference: '' - 24b2947e-a498-4b3a-9b46-44a716915537: !Template + 47b5e0fe-5d5c-4c2a-aa7d-19c6c057eaa6: !Template answer_choices: null - id: 24b2947e-a498-4b3a-9b46-44a716915537 - jinja: 'Translate into Chinese: {{translation["en"]}} + id: 47b5e0fe-5d5c-4c2a-aa7d-19c6c057eaa6 + jinja: ' {{translation["en"]}} translates into Chinese as: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - translate_as_en-zh-target reference: '' - 2b306a49-9307-4dda-bf78-cca4a33e6ae6: !Template + 52a51e61-a646-4c87-9683-d37ba7b1e0ad: !Template answer_choices: null - id: 2b306a49-9307-4dda-bf78-cca4a33e6ae6 - jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 52a51e61-a646-4c87-9683-d37ba7b1e0ad + jinja: '{{translation["en"]}} = Chinese: + + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-source+target - reference: '' - 3c810a32-b8e2-4e96-8b3a-ec7be3051b2c: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 53ab524b-2afa-46a0-adb8-48815fd93468: !Template answer_choices: null - id: 3c810a32-b8e2-4e96-8b3a-ec7be3051b2c - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: 53ab524b-2afa-46a0-adb8-48815fd93468 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target - reference: '' - 462b62aa-4d29-4fe6-b4e5-200c00d6ea4d: !Template + - gpt-3-zh-en-source+target + reference: GPT-3 paper + 545a4ee8-9e7e-4b93-81a9-2a591b7bb0d4: !Template answer_choices: null - id: 462b62aa-4d29-4fe6-b4e5-200c00d6ea4d - jinja: 'How do you say {{translation["zh"]}} in English? + id: 545a4ee8-9e7e-4b93-81a9-2a591b7bb0d4 + jinja: 'What is the Chinese translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target - reference: '' - 49fb6c49-c090-416a-bb2f-de3ccb1f740f: !Template + - gpt-3-en-zh-target + reference: GPT-3 paper + 575b7ead-1786-48fd-8759-6b9a773d42dc: !Template answer_choices: null - id: 49fb6c49-c090-416a-bb2f-de3ccb1f740f - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: 575b7ead-1786-48fd-8759-6b9a773d42dc + jinja: 'How do you say {{translation["zh"]}} in English? - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-source+target - reference: GPT 3 paper - 534893e1-d429-42da-9ab7-a226456f5b99: !Template + - how_to_say-zh-en-target + reference: '' + 5f97369b-a08d-4ded-9ce0-bc86745a9b26: !Template answer_choices: null - id: 534893e1-d429-42da-9ab7-a226456f5b99 - jinja: '{{translation["zh"]}} = English: + id: 5f97369b-a08d-4ded-9ce0-bc86745a9b26 + jinja: '{{translation["zh"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -111,27 +112,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 5e73802f-3c78-444a-ba86-c4eaf811ce7e: !Template + - translate_as_zh-en-target + reference: '' + 6716518e-23f5-4dcd-9f57-108d909b67b7: !Template answer_choices: null - id: 5e73802f-3c78-444a-ba86-c4eaf811ce7e - jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation - is: + id: 6716518e-23f5-4dcd-9f57-108d909b67b7 + jinja: 'Translate this into Chinese: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-target + - translate-en-zh-target reference: '' - 6df05500-04c1-4bcd-b5eb-3e21e3500f77: !Template + 6c15965b-c093-4b8f-a798-d91c6bf84720: !Template answer_choices: null - id: 6df05500-04c1-4bcd-b5eb-3e21e3500f77 - jinja: 'Translate into English: {{translation["zh"]}} + id: 6c15965b-c093-4b8f-a798-d91c6bf84720 + jinja: 'Translate this into English: {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -142,67 +142,65 @@ templates: name: !!python/tuple - translate-zh-en-target reference: '' - 723730ea-9efc-4fc8-9fb3-fdf6cb4d4f04: !Template + 799657ec-3fa5-4764-a281-df90bf698178: !Template answer_choices: null - id: 723730ea-9efc-4fc8-9fb3-fdf6cb4d4f04 - jinja: 'What is the Chinese translation of: {{translation["en"]}} + id: 799657ec-3fa5-4764-a281-df90bf698178 + jinja: 'Translate this from Chinese into English: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-target - reference: GPT-3 paper - 7e1d7244-5d3d-4a22-be30-022df5c4b04e: !Template + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 804991ab-d1ca-4c11-867a-2bf502e97a6b: !Template answer_choices: null - id: 7e1d7244-5d3d-4a22-be30-022df5c4b04e - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: 804991ab-d1ca-4c11-867a-2bf502e97a6b + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target - reference: '' - 8717c806-a408-4c2a-98b3-0caf4e3bf51f: !Template + - gpt-3-en-zh-source+target + reference: GPT 3 paper + 84a1183a-6845-4160-84e1-29cdc9e3ca8d: !Template answer_choices: null - id: 8717c806-a408-4c2a-98b3-0caf4e3bf51f - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Chinese translation is: - - ||| {{translation["zh"]}}' + id: 84a1183a-6845-4160-84e1-29cdc9e3ca8d + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-source+target + - a_good_translation-en-zh-target reference: '' - 99e6cd94-60e3-4550-81a5-1905226f61a1: !Template + 87ad8be4-cc19-439d-b215-63eaab5f0531: !Template answer_choices: null - id: 99e6cd94-60e3-4550-81a5-1905226f61a1 - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: 87ad8be4-cc19-439d-b215-63eaab5f0531 + jinja: 'English: {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target + - translate_as_en-zh-source+target reference: '' - 9b059296-780f-4efc-af83-5a1f9de421c3: !Template + 8a21adc7-a4a3-4f08-9682-4af13dd23ad1: !Template answer_choices: null - id: 9b059296-780f-4efc-af83-5a1f9de421c3 - jinja: 'Translate Chinese into English: {{translation["zh"]}} + id: 8a21adc7-a4a3-4f08-9682-4af13dd23ad1 + jinja: 'What is the English translation of : {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -211,111 +209,113 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 9ca6c947-df2c-40f4-91e7-4e8fa67c3028: !Template + - gpt-3-zh-en-target + reference: GPT-3 paper + 94aad9b2-a0c0-42f6-a9cb-593408dfc4da: !Template answer_choices: null - id: 9ca6c947-df2c-40f4-91e7-4e8fa67c3028 - jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version - should say: + id: 94aad9b2-a0c0-42f6-a9cb-593408dfc4da + jinja: 'Translate this from English into Chinese: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-source+target + - translate-en-zh-source+starget reference: '' - aeb8aeb4-c0f2-40a9-96c6-3e33a7924e83: !Template + 981898d6-e80a-42fb-9b60-980a1cbf2bcd: !Template answer_choices: null - id: aeb8aeb4-c0f2-40a9-96c6-3e33a7924e83 - jinja: ' {{translation["en"]}} translates into Chinese as: + id: 981898d6-e80a-42fb-9b60-980a1cbf2bcd + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-target + - version-zh-en-source+target reference: '' - bf32bab6-3e68-4ffb-b3cb-e2dc30d1260e: !Template + a74116a0-1889-47a4-91a1-26fd5ebcf668: !Template answer_choices: null - id: bf32bab6-3e68-4ffb-b3cb-e2dc30d1260e - jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation - is: ||| {{translation["zh"]}}' + id: a74116a0-1889-47a4-91a1-26fd5ebcf668 + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-target + - a_good_translation-zh-en-source+target reference: '' - cd1ad077-79b7-4799-b065-b899304a3f45: !Template + c489e70f-d9d6-47f0-9ad5-02f0c21a65f9: !Template answer_choices: null - id: cd1ad077-79b7-4799-b065-b899304a3f45 - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: c489e70f-d9d6-47f0-9ad5-02f0c21a65f9 + jinja: 'Chinese: {{translation["zh"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target + - translate_as_zh-en-source+target reference: '' - cdfdad6a-3f99-41af-bc5f-4f852c8c4df6: !Template + c7d75cc8-d7e9-4e12-99b1-308a7cb1c31e: !Template answer_choices: null - id: cdfdad6a-3f99-41af-bc5f-4f852c8c4df6 - jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + id: c7d75cc8-d7e9-4e12-99b1-308a7cb1c31e + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-source+target - reference: GPT-3 paper - df2baa97-9839-45db-bccb-2295cd10e0ac: !Template + - version-en-zh-source+target + reference: '' + cbc8501e-8bdd-4ddf-8b8f-dd9f05d7e152: !Template answer_choices: null - id: df2baa97-9839-45db-bccb-2295cd10e0ac - jinja: 'What is the English translation of : {{translation["zh"]}} + id: cbc8501e-8bdd-4ddf-8b8f-dd9f05d7e152 + jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation + is: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-target - reference: GPT-3 paper - e3f2165c-cae5-41ae-b947-0d90e42c9879: !Template + - a_good_translation-zh-en-target + reference: '' + d08a7b31-7de6-41b9-ac2b-a34b4f7bc23c: !Template answer_choices: null - id: e3f2165c-cae5-41ae-b947-0d90e42c9879 - jinja: 'Chinese: {{translation["zh"]}} = English: + id: d08a7b31-7de6-41b9-ac2b-a34b4f7bc23c + jinja: 'How do you say {{translation["en"]}} in Chinese? - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - e472a8ba-7822-471a-b3ca-39db34e994fc: !Template + - how_to_say-en-zh-target + reference: '' + d7c43184-8622-4d39-88d8-de2097ac5817: !Template answer_choices: null - id: e472a8ba-7822-471a-b3ca-39db34e994fc - jinja: 'English: {{translation["en"]}} = Chinese: + id: d7c43184-8622-4d39-88d8-de2097ac5817 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Chinese translation is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -324,48 +324,48 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - e49a6349-658f-49af-a89a-b48ca1526c97: !Template + - a_good_translation-en-zh-source+target + reference: '' + dacf1578-eb48-42a2-9b4e-3352097b3220: !Template answer_choices: null - id: e49a6349-658f-49af-a89a-b48ca1526c97 - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: dacf1578-eb48-42a2-9b4e-3352097b3220 + jinja: '{{translation["zh"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target - reference: '' - e9abd12f-8bdc-4092-a1ba-c747dea63876: !Template + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f088cd1f-1bc0-4014-ac04-f9c77e1c1d2b: !Template answer_choices: null - id: e9abd12f-8bdc-4092-a1ba-c747dea63876 - jinja: '{{translation["en"]}} = Chinese: + id: f088cd1f-1bc0-4014-ac04-f9c77e1c1d2b + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f66950cc-fdc2-44bd-bee3-3a5d230da47c: !Template + - version-en-zh-target + reference: '' + fe9ed444-6445-4886-ae18-ff96932ae661: !Template answer_choices: null - id: f66950cc-fdc2-44bd-bee3-3a5d230da47c - jinja: 'Translate English into Chinese: {{translation["en"]}} + id: fe9ed444-6445-4886-ae18-ff96932ae661 + jinja: 'Chinese: {{translation["zh"]}} = English: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-source+starget - reference: '' + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt19/de-en/templates.yaml b/promptsource/templates/wmt19/de-en/templates.yaml index a35a55712..8e515fc13 100644 --- a/promptsource/templates/wmt19/de-en/templates.yaml +++ b/promptsource/templates/wmt19/de-en/templates.yaml @@ -1,39 +1,24 @@ dataset: wmt19 subset: de-en templates: - 016f488b-e947-4082-becd-e5ce404ccf71: !Template + 0342b9f3-02bd-4602-94bf-84df1c89fbf7: !Template answer_choices: null - id: 016f488b-e947-4082-becd-e5ce404ccf71 - jinja: 'Translate into English: {{translation["de"]}} + id: 0342b9f3-02bd-4602-94bf-84df1c89fbf7 + jinja: 'Translate this into German: {{translation["en"]}} - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - translate-de-en-target - reference: '' - 0702aabd-283c-455e-bb31-7798d6b6fabe: !Template - answer_choices: null - id: 0702aabd-283c-455e-bb31-7798d6b6fabe - jinja: 'If the original version says: {{translation["de"]}}; then the English - version should say: - - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-target + - translate-en-de-target reference: '' - 16cceacf-8525-4d04-8d83-4fa4cc9c6d1e: !Template + 21cb3e62-e78e-460e-88fd-6895bf1e9faa: !Template answer_choices: null - id: 16cceacf-8525-4d04-8d83-4fa4cc9c6d1e - jinja: 'How do you say {{translation["de"]}} in English? + id: 21cb3e62-e78e-460e-88fd-6895bf1e9faa + jinja: '{{translation["de"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -42,26 +27,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-de-en-target - reference: '' - 1add00c0-9413-45ad-9a2b-17579f04abe6: !Template + - xglm-de-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 365f80d7-5e4f-40e5-8567-f4e60f2f71d9: !Template answer_choices: null - id: 1add00c0-9413-45ad-9a2b-17579f04abe6 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good German translation is: + id: 365f80d7-5e4f-40e5-8567-f4e60f2f71d9 + jinja: 'German: {{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-source+target + - translate_as_de-en-source+target reference: '' - 2c81808d-efc4-4d15-aa72-6ef02591c0a9: !Template + 446c8f26-354b-4be8-ad17-df74e46d5527: !Template answer_choices: null - id: 2c81808d-efc4-4d15-aa72-6ef02591c0a9 + id: 446c8f26-354b-4be8-ad17-df74e46d5527 jinja: 'German: {{translation["de"]}} = English: ||| {{translation["en"]}}' @@ -73,39 +57,37 @@ templates: name: !!python/tuple - xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 323e9cac-0287-4cb3-b8f4-42b91a93e8f4: !Template + 46e30fb6-b626-4846-85ce-8b66cec0eb70: !Template answer_choices: null - id: 323e9cac-0287-4cb3-b8f4-42b91a93e8f4 - jinja: 'Translate English into German: {{translation["en"]}} + id: 46e30fb6-b626-4846-85ce-8b66cec0eb70 + jinja: 'English: {{translation["en"]}} = German: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-de-source+starget - reference: '' - 42017cdc-da85-4c29-bf4e-692d20385c16: !Template + - xglm-en-de-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 51f607b9-09c1-430a-a05c-2c0c29e1f27f: !Template answer_choices: null - id: 42017cdc-da85-4c29-bf4e-692d20385c16 - jinja: 'If the original version says: {{translation["en"]}}; then the German version - should say: - - ||| {{translation["de"]}}' + id: 51f607b9-09c1-430a-a05c-2c0c29e1f27f + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-target + - a_good_translation-de-en-source+target reference: '' - 46272220-d718-4912-885d-b779ba22077a: !Template + 5650e2eb-b31d-4a66-90e5-3dc358b0a152: !Template answer_choices: null - id: 46272220-d718-4912-885d-b779ba22077a - jinja: 'What is the English translation of the German sentence: {{translation["de"]}} + id: 5650e2eb-b31d-4a66-90e5-3dc358b0a152 + jinja: 'What is the English translation of : {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -114,12 +96,13 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-source+target + - gpt-3-de-en-target reference: GPT-3 paper - 48ee7f3a-9a61-4106-9a95-7f9182a96ba1: !Template + 5cc18e97-9267-4e8c-9f89-354e595880a4: !Template answer_choices: null - id: 48ee7f3a-9a61-4106-9a95-7f9182a96ba1 - jinja: 'How do you say {{translation["en"]}} in German? + id: 5cc18e97-9267-4e8c-9f89-354e595880a4 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good German translation is: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -128,151 +111,154 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-de-target + - a_good_translation-en-de-source+target reference: '' - 52177d5b-7b94-41e2-bcd9-143b5aab118e: !Template + 64a199ac-e338-4a03-9879-1f62c2698dcc: !Template answer_choices: null - id: 52177d5b-7b94-41e2-bcd9-143b5aab118e - jinja: '{{translation["en"]}} = German: + id: 64a199ac-e338-4a03-9879-1f62c2698dcc + jinja: 'How do you say {{translation["de"]}} in English? - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 5b09b443-0b71-4406-a4c4-7ca634e18aaf: !Template + - how_to_say-de-en-target + reference: '' + 650edf2a-2ddb-4c41-8868-c7b542acf4ba: !Template answer_choices: null - id: 5b09b443-0b71-4406-a4c4-7ca634e18aaf - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 650edf2a-2ddb-4c41-8868-c7b542acf4ba + jinja: 'Given the following passage: {{translation["en"]}} , a good German translation + is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-source+target + - a_good_translation-en-de-target reference: '' - 5d35fa9e-12ec-452b-8c0a-4b5b21c4b107: !Template + 71386534-dabb-4fb4-88ba-86ae0062cab7: !Template answer_choices: null - id: 5d35fa9e-12ec-452b-8c0a-4b5b21c4b107 - jinja: '{{translation["de"]}} translates into English as: + id: 71386534-dabb-4fb4-88ba-86ae0062cab7 + jinja: 'Translate this into English: {{translation["de"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-target + - translate-de-en-target reference: '' - 5ea19842-e310-4f29-bfe8-40555eb90ecb: !Template + 71a31bbb-3cd8-4755-95b7-ac7f16d47e7b: !Template answer_choices: null - id: 5ea19842-e310-4f29-bfe8-40555eb90ecb - jinja: 'Translate German into English: {{translation["de"]}} + id: 71a31bbb-3cd8-4755-95b7-ac7f16d47e7b + jinja: 'What is the German translation of the English sentence: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 74d50f3e-7ba9-46ad-a417-c155754e37f4: !Template + - gpt-3-en-de-source+target + reference: GPT 3 paper + 78f7787e-47b5-4c9c-afd2-14252b9e006c: !Template answer_choices: null - id: 74d50f3e-7ba9-46ad-a417-c155754e37f4 - jinja: 'English: {{translation["en"]}} translates into German as: + id: 78f7787e-47b5-4c9c-afd2-14252b9e006c + jinja: 'If the original version says: {{translation["en"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-source+target + - version-en-de-target reference: '' - a1d31cb0-c10c-43d8-a272-2e9491b4b36d: !Template + 84e4c8cf-fea3-4b84-9b41-79ea64c1b014: !Template answer_choices: null - id: a1d31cb0-c10c-43d8-a272-2e9491b4b36d - jinja: 'German: {{translation["de"]}} translates into English as: + id: 84e4c8cf-fea3-4b84-9b41-79ea64c1b014 + jinja: 'If the original version says: {{translation["de"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-en-source+target + - version-de-en-target reference: '' - aef2c049-27c2-4786-bc4a-37175592cdf9: !Template + 95191307-4681-4973-aa29-3017ac7023fa: !Template answer_choices: null - id: aef2c049-27c2-4786-bc4a-37175592cdf9 - jinja: 'Given the following passage: {{translation["en"]}} , a good German translation - is: ||| {{translation["de"]}}' + id: 95191307-4681-4973-aa29-3017ac7023fa + jinja: 'Translate this from English into German: {{translation["en"]}} + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-de-target + - translate-en-de-source+starget reference: '' - b2aa1b51-21da-4fc1-80be-d19f12c378d7: !Template + 953b1373-2207-4916-a583-9f69ba5af1ff: !Template answer_choices: null - id: b2aa1b51-21da-4fc1-80be-d19f12c378d7 - jinja: 'Given the following passage: {{translation["de"]}} , a good English translation - is: + id: 953b1373-2207-4916-a583-9f69ba5af1ff + jinja: 'If the German version says: {{translation["de"]}}; then the English version + should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-en-target + - version-de-en-source+target reference: '' - b5c0e8be-bf2f-42d7-84c7-af453179188c: !Template + a0801f73-bba4-46da-a915-26a106d2c4bc: !Template answer_choices: null - id: b5c0e8be-bf2f-42d7-84c7-af453179188c - jinja: ' {{translation["en"]}} translates into German as: + id: a0801f73-bba4-46da-a915-26a106d2c4bc + jinja: '{{translation["de"]}} translates into English as: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-de-target + - translate_as_de-en-target reference: '' - ba6e29d1-9143-4ac2-ba87-aa9043fa2b17: !Template + a8c0831c-d260-4ef6-91e2-6cef49a80858: !Template answer_choices: null - id: ba6e29d1-9143-4ac2-ba87-aa9043fa2b17 - jinja: '{{translation["de"]}} = English: + id: a8c0831c-d260-4ef6-91e2-6cef49a80858 + jinja: 'How do you say {{translation["en"]}} in German? - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c456403b-179d-4653-8e26-bc9fd7b50c2e: !Template + - how_to_say-en-de-target + reference: '' + b324c249-11f6-47e6-8aff-da8f3066a083: !Template answer_choices: null - id: c456403b-179d-4653-8e26-bc9fd7b50c2e - jinja: 'Translate into German: {{translation["en"]}} + id: b324c249-11f6-47e6-8aff-da8f3066a083 + jinja: 'What is the German translation of: {{translation["en"]}} ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -281,12 +267,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-de-target - reference: '' - c7f170d0-f8a9-4a02-a9b5-e8db22a6b76f: !Template + - gpt-3-en-de-target + reference: GPT-3 paper + ba2af4e3-78a6-41b0-bd57-4f58196aa584: !Template answer_choices: null - id: c7f170d0-f8a9-4a02-a9b5-e8db22a6b76f - jinja: 'English: {{translation["en"]}} = German: + id: ba2af4e3-78a6-41b0-bd57-4f58196aa584 + jinja: '{{translation["en"]}} = German: ||| {{translation["de"]}}' metadata: !TemplateMetadata @@ -295,70 +281,70 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-de-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - c843899a-1292-4129-84a1-7783c5d86184: !Template + - xglm-en-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + d111aea3-c014-4114-9bf9-4135e33181c9: !Template answer_choices: null - id: c843899a-1292-4129-84a1-7783c5d86184 - jinja: 'If the English version says: {{translation["en"]}}; then the German version - should say: + id: d111aea3-c014-4114-9bf9-4135e33181c9 + jinja: 'Given the following passage: {{translation["de"]}} , a good English translation + is: - ||| {{translation["de"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-de-source+target + - a_good_translation-de-en-target reference: '' - cad37129-c4f7-4fe7-9697-14ec07ddd78b: !Template + d57cbdfd-3380-4318-8755-7f3e66381e1a: !Template answer_choices: null - id: cad37129-c4f7-4fe7-9697-14ec07ddd78b - jinja: 'What is the German translation of: {{translation["en"]}} + id: d57cbdfd-3380-4318-8755-7f3e66381e1a + jinja: ' {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-target - reference: GPT-3 paper - d80689a8-f4da-4416-9939-3220c40c00aa: !Template + - translate_as_en-de-target + reference: '' + e10830ef-4e58-4d17-b5e0-20961b960b95: !Template answer_choices: null - id: d80689a8-f4da-4416-9939-3220c40c00aa - jinja: 'What is the German translation of the English sentence: {{translation["en"]}} + id: e10830ef-4e58-4d17-b5e0-20961b960b95 + jinja: 'English: {{translation["en"]}} translates into German as: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-de-source+target - reference: GPT 3 paper - e0b2bece-f655-4279-997b-e893b94b04eb: !Template + - translate_as_en-de-source+target + reference: '' + e3c62d5c-373d-45ca-a840-52ad4d6609d4: !Template answer_choices: null - id: e0b2bece-f655-4279-997b-e893b94b04eb - jinja: 'If the German version says: {{translation["de"]}}; then the English version + id: e3c62d5c-373d-45ca-a840-52ad4d6609d4 + jinja: 'If the English version says: {{translation["en"]}}; then the German version should say: - ||| {{translation["en"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-en-source+target + - version-en-de-source+target reference: '' - f5accc29-205f-430b-a74e-54434b59b6f6: !Template + e3cacbbb-763f-43a4-9ca1-e05d0e914a14: !Template answer_choices: null - id: f5accc29-205f-430b-a74e-54434b59b6f6 - jinja: 'What is the English translation of : {{translation["de"]}} + id: e3cacbbb-763f-43a4-9ca1-e05d0e914a14 + jinja: 'What is the English translation of the German sentence: {{translation["de"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -367,5 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-en-target + - gpt-3-de-en-source+target reference: GPT-3 paper + e9816ae4-4c1c-4c7b-a0a4-5e63bbea6936: !Template + answer_choices: null + id: e9816ae4-4c1c-4c7b-a0a4-5e63bbea6936 + jinja: 'Translate this from German into English: {{translation["de"]}} + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - translate-de-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf diff --git a/promptsource/templates/wmt19/fi-en/templates.yaml b/promptsource/templates/wmt19/fi-en/templates.yaml index 34950b0d5..18002b055 100644 --- a/promptsource/templates/wmt19/fi-en/templates.yaml +++ b/promptsource/templates/wmt19/fi-en/templates.yaml @@ -1,25 +1,25 @@ dataset: wmt19 subset: fi-en templates: - 023d23ee-81ea-4257-bd52-61bf657498a8: !Template + 04598a46-5fb2-4e2d-9174-931b9033d694: !Template answer_choices: null - id: 023d23ee-81ea-4257-bd52-61bf657498a8 - jinja: 'How do you say {{translation["en"]}} in Finnish? + id: 04598a46-5fb2-4e2d-9174-931b9033d694 + jinja: 'Translate this into English: {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-fi-target + - translate-fi-en-target reference: '' - 0a833174-933a-447f-b5bc-84f2f9fb7966: !Template + 05cd2f4d-c21d-47a0-b2c7-bc048b23d6e7: !Template answer_choices: null - id: 0a833174-933a-447f-b5bc-84f2f9fb7966 - jinja: 'If the original version says: {{translation["fi"]}}; then the English - version should say: + id: 05cd2f4d-c21d-47a0-b2c7-bc048b23d6e7 + jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version + should say: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -28,210 +28,211 @@ templates: - BLEU original_task: true name: !!python/tuple - - version-fi-en-target + - version-fi-en-source+target reference: '' - 17ecce9f-aa67-4735-abac-79727fd8ba87: !Template + 07140e32-85f0-4562-b61f-0b42480b4740: !Template answer_choices: null - id: 17ecce9f-aa67-4735-abac-79727fd8ba87 - jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation - is: + id: 07140e32-85f0-4562-b61f-0b42480b4740 + jinja: 'If the original version says: {{translation["fi"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-target + - version-fi-en-target reference: '' - 22fb64db-ff31-4dbb-a88b-f986af30af07: !Template + 0a8b9a3f-e108-48e3-ba56-26298c529e39: !Template answer_choices: null - id: 22fb64db-ff31-4dbb-a88b-f986af30af07 - jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation - is: ||| {{translation["fi"]}}' + id: 0a8b9a3f-e108-48e3-ba56-26298c529e39 + jinja: 'Translate this into Finnish: {{translation["en"]}} + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-target + - translate-en-fi-target reference: '' - 250690cf-c5c0-4b08-adea-dcebefd9182e: !Template + 18d3b42e-dd08-4ff4-aaca-ae33aae6262a: !Template answer_choices: null - id: 250690cf-c5c0-4b08-adea-dcebefd9182e - jinja: 'Translate English into Finnish: {{translation["en"]}} + id: 18d3b42e-dd08-4ff4-aaca-ae33aae6262a + jinja: 'If the original version says: {{translation["en"]}}; then the Finnish + version should say: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-source+starget + - version-en-fi-target reference: '' - 3d74225c-cd35-4501-bee2-54023e085209: !Template + 222806b8-6b75-4ebd-99aa-9b444448606d: !Template answer_choices: null - id: 3d74225c-cd35-4501-bee2-54023e085209 - jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} + id: 222806b8-6b75-4ebd-99aa-9b444448606d + jinja: 'English: {{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-source+target - reference: GPT-3 paper - 51ab7868-5f60-49b6-91d5-42b01dc8ee78: !Template + - xglm-en-fi-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 228fedd6-f8e2-4b22-9d88-5dbd159121f0: !Template answer_choices: null - id: 51ab7868-5f60-49b6-91d5-42b01dc8ee78 - jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version - should say: + id: 228fedd6-f8e2-4b22-9d88-5dbd159121f0 + jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-source+target - reference: '' - 53fce238-a0e0-43a0-8376-d5f652284024: !Template + - gpt-3-en-fi-source+target + reference: GPT 3 paper + 3795be9e-22af-4a75-864a-49203df483f5: !Template answer_choices: null - id: 53fce238-a0e0-43a0-8376-d5f652284024 - jinja: '{{translation["en"]}} = Finnish: + id: 3795be9e-22af-4a75-864a-49203df483f5 + jinja: 'What is the English translation of : {{translation["fi"]}} - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 6a1c3d44-adb7-415b-bdcc-b36a10892432: !Template + - gpt-3-fi-en-target + reference: GPT-3 paper + 4d491547-9bc5-4197-a5da-8bd4fe36b5fb: !Template answer_choices: null - id: 6a1c3d44-adb7-415b-bdcc-b36a10892432 - jinja: 'If the Finnish version says: {{translation["fi"]}}; then the English version - should say: + id: 4d491547-9bc5-4197-a5da-8bd4fe36b5fb + jinja: 'Finnish: {{translation["fi"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fi-en-source+target + - translate_as_fi-en-source+target reference: '' - 7497607e-de26-48bd-b679-453ac7eaa3f5: !Template + 50ae69bf-ee4c-4c08-80aa-0c237080a184: !Template answer_choices: null - id: 7497607e-de26-48bd-b679-453ac7eaa3f5 - jinja: 'Translate into English: {{translation["fi"]}} + id: 50ae69bf-ee4c-4c08-80aa-0c237080a184 + jinja: 'Finnish: {{translation["fi"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-target - reference: '' - 7ab56181-5880-4ec0-a3a3-a080a3febbcc: !Template + - xglm-fi-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 50b8f1a8-9b72-45b2-9e76-47a1cae9b05b: !Template answer_choices: null - id: 7ab56181-5880-4ec0-a3a3-a080a3febbcc - jinja: 'Finnish: {{translation["fi"]}} = English: + id: 50b8f1a8-9b72-45b2-9e76-47a1cae9b05b + jinja: 'If the English version says: {{translation["en"]}}; then the Finnish version + should say: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 7b10fd1d-d982-4fb6-a49e-31c2611a3e59: !Template + - version-en-fi-source+target + reference: '' + 50e28f97-014f-4e94-a530-bcdc7b7689d6: !Template answer_choices: null - id: 7b10fd1d-d982-4fb6-a49e-31c2611a3e59 - jinja: 'Translate Finnish into English: {{translation["fi"]}} + id: 50e28f97-014f-4e94-a530-bcdc7b7689d6 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Finnish translation is: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fi-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 8476f748-b0a8-428e-b817-5178744d920e: !Template + - a_good_translation-en-fi-source+target + reference: '' + 5acb2abe-33c6-4304-853f-4e7f94143bc3: !Template answer_choices: null - id: 8476f748-b0a8-428e-b817-5178744d920e - jinja: 'English: {{translation["en"]}} = Finnish: + id: 5acb2abe-33c6-4304-853f-4e7f94143bc3 + jinja: 'How do you say {{translation["fi"]}} in English? - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-fi-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 88d5b90d-d1e4-4966-8598-74677f046168: !Template + - how_to_say-fi-en-target + reference: '' + 5dc46f7a-bf1e-48ae-9098-20b9e4a20798: !Template answer_choices: null - id: 88d5b90d-d1e4-4966-8598-74677f046168 - jinja: '{{translation["fi"]}} = English: + id: 5dc46f7a-bf1e-48ae-9098-20b9e4a20798 + jinja: '{{translation["en"]}} = Finnish: - ||| {{translation["en"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fi-en-target + - xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9218230a-6d3b-4a76-8580-7c5c01ee3ee1: !Template + 6107fac1-bb73-405c-b60f-06bd20369b14: !Template answer_choices: null - id: 9218230a-6d3b-4a76-8580-7c5c01ee3ee1 - jinja: 'How do you say {{translation["fi"]}} in English? + id: 6107fac1-bb73-405c-b60f-06bd20369b14 + jinja: 'What is the English translation of the Finnish sentence: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-fi-en-target - reference: '' - 92599ff2-6d98-487c-adae-9edba157a26f: !Template + - gpt-3-fi-en-source+target + reference: GPT-3 paper + 679d6be7-8b8a-4dc4-b6bf-40f89898608b: !Template answer_choices: null - id: 92599ff2-6d98-487c-adae-9edba157a26f - jinja: 'English: {{translation["en"]}} translates into Finnish as: - - ||| {{translation["fi"]}}' + id: 679d6be7-8b8a-4dc4-b6bf-40f89898608b + jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-source+target + - a_good_translation-fi-en-source+target reference: '' - 95674532-7c32-48fd-bc7b-4b097504a35d: !Template + 7076f205-f4f8-4993-b28c-3f3a1e002479: !Template answer_choices: null - id: 95674532-7c32-48fd-bc7b-4b097504a35d - jinja: ' {{translation["en"]}} translates into Finnish as: + id: 7076f205-f4f8-4993-b28c-3f3a1e002479 + jinja: 'How do you say {{translation["en"]}} in Finnish? ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -240,27 +241,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-fi-target + - how_to_say-en-fi-target reference: '' - 9bed380b-ae73-41a0-9e83-c5ee1ad04af2: !Template + 7afceb71-777a-4499-abc3-96af3a28447c: !Template answer_choices: null - id: 9bed380b-ae73-41a0-9e83-c5ee1ad04af2 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Finnish translation is: + id: 7afceb71-777a-4499-abc3-96af3a28447c + jinja: 'Translate this from English into Finnish: {{translation["en"]}} - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-fi-source+target + - translate-en-fi-source+starget reference: '' - c1ec4708-85a5-4b30-be8f-618f983465e7: !Template + 7db8e1b5-3d2b-4e49-b710-001e86451e2e: !Template answer_choices: null - id: c1ec4708-85a5-4b30-be8f-618f983465e7 - jinja: '{{translation["fi"]}} translates into English as: + id: 7db8e1b5-3d2b-4e49-b710-001e86451e2e + jinja: 'Given the following passage: {{translation["fi"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -269,40 +270,40 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-target + - a_good_translation-fi-en-target reference: '' - c24a9fdb-86c6-48ad-946e-085688445f3a: !Template + 7f60153a-78fb-4658-ba0c-30f31c82e2eb: !Template answer_choices: null - id: c24a9fdb-86c6-48ad-946e-085688445f3a - jinja: 'Given the following source text in Finnish: {{translation["fi"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 7f60153a-78fb-4658-ba0c-30f31c82e2eb + jinja: 'English: {{translation["en"]}} translates into Finnish as: + + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fi-en-source+target + - translate_as_en-fi-source+target reference: '' - c87d7a83-fb7d-461f-b43d-7ee5aecef31f: !Template + a5e5fb83-1bc3-439e-b001-11269a3013e2: !Template answer_choices: null - id: c87d7a83-fb7d-461f-b43d-7ee5aecef31f - jinja: 'If the original version says: {{translation["en"]}}; then the Finnish - version should say: + id: a5e5fb83-1bc3-439e-b001-11269a3013e2 + jinja: '{{translation["fi"]}} = English: - ||| {{translation["fi"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-fi-target - reference: '' - c8f66d73-f7d8-42d0-8207-332b73389354: !Template + - xglm-fi-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + bdb3e1cf-723b-4dad-90a0-5e00591b48dd: !Template answer_choices: null - id: c8f66d73-f7d8-42d0-8207-332b73389354 - jinja: 'Translate into Finnish: {{translation["en"]}} + id: bdb3e1cf-723b-4dad-90a0-5e00591b48dd + jinja: 'What is the Finnish translation of: {{translation["en"]}} ||| {{translation["fi"]}}' metadata: !TemplateMetadata @@ -311,61 +312,60 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-fi-target - reference: '' - d515992b-4aee-4af8-b2b9-528d02b9915d: !Template + - gpt-3-en-fi-target + reference: GPT-3 paper + d0f69c91-295a-4e9a-b129-18b5338bf152: !Template answer_choices: null - id: d515992b-4aee-4af8-b2b9-528d02b9915d - jinja: 'What is the Finnish translation of the English sentence: {{translation["en"]}} - - ||| {{translation["fi"]}}' + id: d0f69c91-295a-4e9a-b129-18b5338bf152 + jinja: 'Given the following passage: {{translation["en"]}} , a good Finnish translation + is: ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-source+target - reference: GPT 3 paper - de8d78ef-955c-4986-9223-ca3ff7a07aa6: !Template + - a_good_translation-en-fi-target + reference: '' + d8523015-12ec-4fd3-a22f-3a745c07a0b3: !Template answer_choices: null - id: de8d78ef-955c-4986-9223-ca3ff7a07aa6 - jinja: 'Finnish: {{translation["fi"]}} translates into English as: + id: d8523015-12ec-4fd3-a22f-3a745c07a0b3 + jinja: 'Translate this from Finnish into English: {{translation["fi"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fi-en-source+target - reference: '' - eeb1c1bd-de2e-4b54-883a-ab2cd9cd40ab: !Template + - translate-fi-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + e463a8f4-da3c-4355-aae3-b23048fc1afb: !Template answer_choices: null - id: eeb1c1bd-de2e-4b54-883a-ab2cd9cd40ab - jinja: 'What is the Finnish translation of: {{translation["en"]}} + id: e463a8f4-da3c-4355-aae3-b23048fc1afb + jinja: ' {{translation["en"]}} translates into Finnish as: - ||| {{translation["fi"]}}' + ||| {{translation["fi"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-fi-target - reference: GPT-3 paper - fcb42e40-d756-401a-b021-2471b6342147: !Template + - translate_as_en-fi-target + reference: '' + ea43fbc9-3239-4ff2-82b9-9f2133d904fb: !Template answer_choices: null - id: fcb42e40-d756-401a-b021-2471b6342147 - jinja: 'What is the English translation of : {{translation["fi"]}} + id: ea43fbc9-3239-4ff2-82b9-9f2133d904fb + jinja: '{{translation["fi"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fi-en-target - reference: GPT-3 paper + - translate_as_fi-en-target + reference: '' diff --git a/promptsource/templates/wmt19/fr-de/templates.yaml b/promptsource/templates/wmt19/fr-de/templates.yaml index c6beb38c6..bbe284cd8 100644 --- a/promptsource/templates/wmt19/fr-de/templates.yaml +++ b/promptsource/templates/wmt19/fr-de/templates.yaml @@ -1,136 +1,151 @@ dataset: wmt19 subset: fr-de templates: - 04f6d194-35a5-45f6-b2ba-edb54ad01956: !Template + 02b79c10-0a8e-4482-9a00-743c9a3666ee: !Template answer_choices: null - id: 04f6d194-35a5-45f6-b2ba-edb54ad01956 - jinja: '{{translation["fr"]}} translates into German as: + id: 02b79c10-0a8e-4482-9a00-743c9a3666ee + jinja: '{{translation["de"]}} = French: - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-de-target + - xglm-de-fr-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 29a91408-85f4-4a55-ae0f-16338b5749a5: !Template + answer_choices: null + id: 29a91408-85f4-4a55-ae0f-16338b5749a5 + jinja: 'Given the following passage: {{translation["de"]}} , a good French translation + is: ||| {{translation["fr"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-de-fr-target reference: '' - 058c6082-3e5b-4c85-bf20-ba43f38947e4: !Template + 2c9b4ab7-9c57-4525-969f-b951df2b2a94: !Template answer_choices: null - id: 058c6082-3e5b-4c85-bf20-ba43f38947e4 - jinja: 'Translate French into German: {{translation["fr"]}} + id: 2c9b4ab7-9c57-4525-969f-b951df2b2a94 + jinja: 'What is the French translation of: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-de-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 09aa2188-5940-4e98-8758-ef1674c28299: !Template + - gpt-3-de-fr-target + reference: GPT-3 paper + 2e3e5a7c-b484-49c2-9660-461da0264f5e: !Template answer_choices: null - id: 09aa2188-5940-4e98-8758-ef1674c28299 - jinja: 'If the original version says: {{translation["de"]}}; then the French version - should say: + id: 2e3e5a7c-b484-49c2-9660-461da0264f5e + jinja: 'German: {{translation["de"]}} translates into French as: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-fr-target + - translate_as_de-fr-source+target reference: '' - 0c05ce7b-e7c4-45fc-bcbd-890050290931: !Template + 3dfc0e89-8459-4498-b5ef-87fe9682d516: !Template answer_choices: null - id: 0c05ce7b-e7c4-45fc-bcbd-890050290931 - jinja: 'What is the German translation of the French sentence: {{translation["fr"]}} + id: 3dfc0e89-8459-4498-b5ef-87fe9682d516 + jinja: 'If the original version says: {{translation["fr"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-de-source+target - reference: GPT-3 paper - 2605ab99-d28e-41c0-b5e4-6f5017f3929b: !Template + - version-fr-de-target + reference: '' + 4101e307-ca38-4848-acd9-616d60e1b8dd: !Template answer_choices: null - id: 2605ab99-d28e-41c0-b5e4-6f5017f3929b - jinja: 'German: {{translation["de"]}} translates into French as: + id: 4101e307-ca38-4848-acd9-616d60e1b8dd + jinja: 'If the German version says: {{translation["de"]}}; then the French version + should say: - ||| {{translation["fr"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_de-fr-source+target + - version-de-fr-source+target reference: '' - 282dab4a-90b8-4376-90e7-15ee3dac65cc: !Template + 44665ae6-6ca1-4e4e-9469-0ff1add72fd1: !Template answer_choices: null - id: 282dab4a-90b8-4376-90e7-15ee3dac65cc - jinja: 'Given the following source text in French: {{translation["fr"]}} , a good - German translation is: ||| {{translation["de"]}}' + id: 44665ae6-6ca1-4e4e-9469-0ff1add72fd1 + jinja: 'Given the following source text in German: {{translation["de"]}} , a good + French translation is: + + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-de-source+target + - a_good_translation-de-fr-source+target reference: '' - 2f0109ed-d052-438c-9475-1252eafb1597: !Template + 4718d5fd-333a-453a-ab9e-ade6c0eb23e9: !Template answer_choices: null - id: 2f0109ed-d052-438c-9475-1252eafb1597 - jinja: '{{translation["de"]}} = French: + id: 4718d5fd-333a-453a-ab9e-ade6c0eb23e9 + jinja: 'French: {{translation["fr"]}} = German: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-de-fr-target + - xglm-fr-de-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - 38776fc6-7e09-4250-923e-863d438833c0: !Template + 538dc918-e66b-444a-bb7b-62e73934bb02: !Template answer_choices: null - id: 38776fc6-7e09-4250-923e-863d438833c0 - jinja: 'What is the French translation of: {{translation["de"]}} + id: 538dc918-e66b-444a-bb7b-62e73934bb02 + jinja: '{{translation["fr"]}} = German: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-fr-target - reference: GPT-3 paper - 42a03567-18d2-41fe-ad01-2aae801c0eb8: !Template + - xglm-fr-de-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 576d28f5-4785-42f2-9431-b33026c43498: !Template answer_choices: null - id: 42a03567-18d2-41fe-ad01-2aae801c0eb8 - jinja: 'If the German version says: {{translation["de"]}}; then the French version - should say: + id: 576d28f5-4785-42f2-9431-b33026c43498 + jinja: 'What is the German translation of the French sentence: {{translation["fr"]}} - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-de-fr-source+target - reference: '' - 430d0b3a-073f-4648-ba42-4705a87298ed: !Template + - gpt-3-fr-de-source+target + reference: GPT-3 paper + 5ead1818-8c5d-4516-89b8-34b10c8371a7: !Template answer_choices: null - id: 430d0b3a-073f-4648-ba42-4705a87298ed + id: 5ead1818-8c5d-4516-89b8-34b10c8371a7 jinja: 'How do you say {{translation["fr"]}} in German? ||| {{translation["de"]}}' @@ -142,10 +157,10 @@ templates: name: !!python/tuple - how_to_say-fr-de-target reference: '' - 51f9edfd-ce4c-481c-923a-88346fd5c58c: !Template + 70631159-fef5-42fb-a154-019fb0d96cd4: !Template answer_choices: null - id: 51f9edfd-ce4c-481c-923a-88346fd5c58c - jinja: 'What is the French translation of the German sentence: {{translation["de"]}} + id: 70631159-fef5-42fb-a154-019fb0d96cd4 + jinja: 'Translate this from German into French: {{translation["de"]}} ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -154,24 +169,25 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-de-fr-source+target - reference: GPT 3 paper - 53d700f4-692a-462d-803a-458cb94d8315: !Template + - translate-de-fr-source+starget + reference: '' + 72bd10ed-0fd9-4718-b175-c20c5bd9a70e: !Template answer_choices: null - id: 53d700f4-692a-462d-803a-458cb94d8315 - jinja: 'Given the following passage: {{translation["de"]}} , a good French translation - is: ||| {{translation["fr"]}}' + id: 72bd10ed-0fd9-4718-b175-c20c5bd9a70e + jinja: 'French: {{translation["fr"]}} translates into German as: + + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-fr-target + - translate_as_fr-de-source+target reference: '' - 7ba506d6-151b-4cf5-8b3d-659886f7ccd4: !Template + 758d7918-2931-4883-9b1c-aeb44227a6c4: !Template answer_choices: null - id: 7ba506d6-151b-4cf5-8b3d-659886f7ccd4 + id: 758d7918-2931-4883-9b1c-aeb44227a6c4 jinja: 'How do you say {{translation["de"]}} in French? ||| {{translation["fr"]}}' @@ -183,24 +199,25 @@ templates: name: !!python/tuple - how_to_say-de-fr-target reference: '' - 99aa7622-989e-4486-82e4-f3708886bcc1: !Template + 7974a5c0-0b58-4067-9c3f-4ef8be35a987: !Template answer_choices: null - id: 99aa7622-989e-4486-82e4-f3708886bcc1 - jinja: 'Translate into German: {{translation["fr"]}} + id: 7974a5c0-0b58-4067-9c3f-4ef8be35a987 + jinja: 'Given the following passage: {{translation["fr"]}} , a good German translation + is: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-fr-de-target + - a_good_translation-fr-de-target reference: '' - ab18ee6c-7b28-4d95-91fc-74d788166709: !Template + 80c1e81e-8e63-478d-bc72-adfce1f95016: !Template answer_choices: null - id: ab18ee6c-7b28-4d95-91fc-74d788166709 - jinja: 'German: {{translation["de"]}} = French: + id: 80c1e81e-8e63-478d-bc72-adfce1f95016 + jinja: ' {{translation["de"]}} translates into French as: ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -209,42 +226,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-de-fr-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - adc7ebbc-8dd3-4c63-8341-a4787f9ab0f6: !Template + - translate_as_de-fr-target + reference: '' + 844bacf0-2da2-4ffd-947d-8898145b6b09: !Template answer_choices: null - id: adc7ebbc-8dd3-4c63-8341-a4787f9ab0f6 - jinja: 'If the original version says: {{translation["fr"]}}; then the German version - should say: + id: 844bacf0-2da2-4ffd-947d-8898145b6b09 + jinja: 'Translate this into French: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-fr-de-target + - translate-de-fr-target reference: '' - b2c178f3-e4ba-4b17-ac90-7145a3986dc6: !Template + 85a1dd29-152e-4064-b97a-2af772ddeb55: !Template answer_choices: null - id: b2c178f3-e4ba-4b17-ac90-7145a3986dc6 - jinja: 'Given the following passage: {{translation["fr"]}} , a good German translation - is: + id: 85a1dd29-152e-4064-b97a-2af772ddeb55 + jinja: 'If the French version says: {{translation["fr"]}}; then the German version + should say: - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-fr-de-target + - version-fr-de-source+target reference: '' - c22ff3b0-df43-49b5-960b-51edc1b0e604: !Template + 871db540-18be-4cc4-818d-9009ce8de896: !Template answer_choices: null - id: c22ff3b0-df43-49b5-960b-51edc1b0e604 - jinja: ' {{translation["de"]}} translates into French as: + id: 871db540-18be-4cc4-818d-9009ce8de896 + jinja: 'German: {{translation["de"]}} = French: ||| {{translation["fr"]}}' metadata: !TemplateMetadata @@ -253,119 +269,103 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_de-fr-target - reference: '' - c3d464e6-2c71-4deb-809b-330491a92438: !Template + - xglm-de-fr-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 92d8b55a-6267-4fa0-af26-f99b921444c5: !Template answer_choices: null - id: c3d464e6-2c71-4deb-809b-330491a92438 - jinja: 'French: {{translation["fr"]}} = German: + id: 92d8b55a-6267-4fa0-af26-f99b921444c5 + jinja: 'Translate this from French into German: {{translation["fr"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fr-de-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c3ed4191-deb0-4be0-9a06-479cced9a1c0: !Template + - translate-fr-de-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 96ab70c9-ad4a-4bd2-863b-8c8aaa27e881: !Template answer_choices: null - id: c3ed4191-deb0-4be0-9a06-479cced9a1c0 - jinja: '{{translation["fr"]}} = German: + id: 96ab70c9-ad4a-4bd2-863b-8c8aaa27e881 + jinja: 'What is the German translation of : {{translation["fr"]}} - ||| {{translation["de"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-fr-de-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c480958d-147f-4fa6-af10-d91551f1896e: !Template + - gpt-3-fr-de-target + reference: GPT-3 paper + a855a184-a86c-4824-920b-da6cf3fe8faf: !Template answer_choices: null - id: c480958d-147f-4fa6-af10-d91551f1896e - jinja: 'Translate German into French: {{translation["de"]}} + id: a855a184-a86c-4824-920b-da6cf3fe8faf + jinja: 'Translate this into German: {{translation["fr"]}} - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-fr-source+starget + - translate-fr-de-target reference: '' - cf3caf39-8aa6-44ea-8586-7879d5fe5dec: !Template + ade41a91-0578-4ce3-ba1a-3cb4537cd266: !Template answer_choices: null - id: cf3caf39-8aa6-44ea-8586-7879d5fe5dec - jinja: 'If the French version says: {{translation["fr"]}}; then the German version + id: ade41a91-0578-4ce3-ba1a-3cb4537cd266 + jinja: 'If the original version says: {{translation["de"]}}; then the French version should say: - ||| {{translation["de"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - version-fr-de-source+target - reference: '' - d5dd5907-05c5-4dec-bfa3-64036e63c1c9: !Template - answer_choices: null - id: d5dd5907-05c5-4dec-bfa3-64036e63c1c9 - jinja: 'French: {{translation["fr"]}} translates into German as: - - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_fr-de-source+target + - version-de-fr-target reference: '' - f00b8dab-a49f-4a22-bd61-e8fe10221a98: !Template + c9332f62-6328-4aec-805b-4fded0df7ffa: !Template answer_choices: null - id: f00b8dab-a49f-4a22-bd61-e8fe10221a98 - jinja: 'Translate into French: {{translation["de"]}} - - ||| {{translation["fr"]}}' + id: c9332f62-6328-4aec-805b-4fded0df7ffa + jinja: 'Given the following source text in French: {{translation["fr"]}} , a good + German translation is: ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-de-fr-target + - a_good_translation-fr-de-source+target reference: '' - f2bed3fa-546e-4d13-ab70-af3fa05ea384: !Template + d1e73e14-b677-4cec-b43f-497f4997747c: !Template answer_choices: null - id: f2bed3fa-546e-4d13-ab70-af3fa05ea384 - jinja: 'Given the following source text in German: {{translation["de"]}} , a good - French translation is: + id: d1e73e14-b677-4cec-b43f-497f4997747c + jinja: '{{translation["fr"]}} translates into German as: - ||| {{translation["fr"]}}' + ||| {{translation["de"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-de-fr-source+target + - translate_as_fr-de-target reference: '' - fedb1953-e3cf-4c48-aa78-f10cdef832fc: !Template + d6b79b6b-be44-4494-be72-513904736115: !Template answer_choices: null - id: fedb1953-e3cf-4c48-aa78-f10cdef832fc - jinja: 'What is the German translation of : {{translation["fr"]}} + id: d6b79b6b-be44-4494-be72-513904736115 + jinja: 'What is the French translation of the German sentence: {{translation["de"]}} - ||| {{translation["de"]}}' + ||| {{translation["fr"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-fr-de-target - reference: GPT-3 paper + - gpt-3-de-fr-source+target + reference: GPT 3 paper diff --git a/promptsource/templates/wmt19/gu-en/templates.yaml b/promptsource/templates/wmt19/gu-en/templates.yaml index 6692ad3ec..65a42f9b0 100644 --- a/promptsource/templates/wmt19/gu-en/templates.yaml +++ b/promptsource/templates/wmt19/gu-en/templates.yaml @@ -1,39 +1,39 @@ dataset: wmt19 subset: gu-en templates: - 3356cbf5-bab8-48b8-8930-aa27a0e428e4: !Template + 00672ffa-2f48-411b-a52c-fa7abf359b68: !Template answer_choices: null - id: 3356cbf5-bab8-48b8-8930-aa27a0e428e4 - jinja: 'Translate into Gurajati: {{translation["en"]}} + id: 00672ffa-2f48-411b-a52c-fa7abf359b68 + jinja: 'If the original version says: {{translation["gu"]}}; then the English + version should say: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-gu-target + - version-gu-en-target reference: '' - 39cb6dd4-1b9e-45bc-aeb5-b43066a84e0f: !Template + 11ba3712-b335-4c47-bcb6-c6cd19203981: !Template answer_choices: null - id: 39cb6dd4-1b9e-45bc-aeb5-b43066a84e0f - jinja: 'Given the following passage: {{translation["gu"]}} , a good English translation - is: + id: 11ba3712-b335-4c47-bcb6-c6cd19203981 + jinja: 'How do you say {{translation["en"]}} in Gurajati? - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-gu-en-target + - how_to_say-en-gu-target reference: '' - 41a4a411-56dc-4e2f-b616-6f41a2e7cd3b: !Template + 28443716-ea33-458c-8c9a-f055dd3a161a: !Template answer_choices: null - id: 41a4a411-56dc-4e2f-b616-6f41a2e7cd3b - jinja: 'What is the Gurajati translation of: {{translation["en"]}} + id: 28443716-ea33-458c-8c9a-f055dd3a161a + jinja: 'What is the Gurajati translation of the English sentence: {{translation["en"]}} ||| {{translation["gu"]}}' metadata: !TemplateMetadata @@ -42,40 +42,39 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-gu-target - reference: GPT-3 paper - 4f119283-2149-4fc5-a0e2-947a30ca0b6e: !Template + - gpt-3-en-gu-source+target + reference: GPT 3 paper + 2e6c5141-6937-49f2-810e-4eb9bd9c4d3c: !Template answer_choices: null - id: 4f119283-2149-4fc5-a0e2-947a30ca0b6e - jinja: 'If the English version says: {{translation["en"]}}; then the Gurajati - version should say: + id: 2e6c5141-6937-49f2-810e-4eb9bd9c4d3c + jinja: 'English: {{translation["en"]}} = Gurajati: - ||| {{translation["gu"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-gu-source+target - reference: '' - 5171b947-ab61-4cbf-8708-b6525bef6444: !Template + - xglm-en-gu-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 3881054b-1e24-4300-b219-4f870bd72de9: !Template answer_choices: null - id: 5171b947-ab61-4cbf-8708-b6525bef6444 - jinja: 'Translate into English: {{translation["gu"]}} + id: 3881054b-1e24-4300-b219-4f870bd72de9 + jinja: 'Translate this from English into Gurajati: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-gu-en-target + - translate-en-gu-source+starget reference: '' - 6441d5dd-b04b-4060-9644-4e57e6f1cd90: !Template + 504f3217-bec3-4015-8209-8ad7676ecb20: !Template answer_choices: null - id: 6441d5dd-b04b-4060-9644-4e57e6f1cd90 + id: 504f3217-bec3-4015-8209-8ad7676ecb20 jinja: 'If the original version says: {{translation["en"]}}; then the Gurajati version should say: @@ -88,77 +87,79 @@ templates: name: !!python/tuple - version-en-gu-target reference: '' - 684c70ae-f44f-4fb3-b387-67c6e2f46506: !Template + 55b2fd44-a3b0-404e-a687-f61d6401b155: !Template answer_choices: null - id: 684c70ae-f44f-4fb3-b387-67c6e2f46506 - jinja: 'Given the following source text in Gurajati: {{translation["gu"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: 55b2fd44-a3b0-404e-a687-f61d6401b155 + jinja: 'Gurajati: {{translation["gu"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-gu-en-source+target + - translate_as_gu-en-source+target reference: '' - 697fba8f-c487-4dbe-a39c-4c3dee1be014: !Template + 5b3df10f-7504-43c6-a667-fc98b72a1dd2: !Template answer_choices: null - id: 697fba8f-c487-4dbe-a39c-4c3dee1be014 - jinja: 'What is the Gurajati translation of the English sentence: {{translation["en"]}} + id: 5b3df10f-7504-43c6-a667-fc98b72a1dd2 + jinja: 'Translate this into English: {{translation["gu"]}} - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-gu-source+target - reference: GPT 3 paper - 710d1d8e-d27a-4360-b77c-557e7ef6f147: !Template + - translate-gu-en-target + reference: '' + 5ff2be23-ef8b-489f-bb43-87a417afdf9b: !Template answer_choices: null - id: 710d1d8e-d27a-4360-b77c-557e7ef6f147 - jinja: 'Given the following passage: {{translation["en"]}} , a good Gurajati translation - is: ||| {{translation["gu"]}}' + id: 5ff2be23-ef8b-489f-bb43-87a417afdf9b + jinja: 'English: {{translation["en"]}} translates into Gurajati as: + + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-gu-target + - translate_as_en-gu-source+target reference: '' - 7837b4a7-3de5-4e15-ac1a-35c2e7850a0e: !Template + 6fcb810b-1f74-4d01-b065-a7a1ffd3a1ce: !Template answer_choices: null - id: 7837b4a7-3de5-4e15-ac1a-35c2e7850a0e - jinja: 'English: {{translation["en"]}} translates into Gurajati as: + id: 6fcb810b-1f74-4d01-b065-a7a1ffd3a1ce + jinja: '{{translation["gu"]}} = English: - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-gu-source+target - reference: '' - 7edc2e1e-e96b-4012-b90a-9e37d075173c: !Template + - xglm-gu-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 79ce60b6-a9aa-4499-bf1e-be4ba040739b: !Template answer_choices: null - id: 7edc2e1e-e96b-4012-b90a-9e37d075173c - jinja: 'Gurajati: {{translation["gu"]}} translates into English as: + id: 79ce60b6-a9aa-4499-bf1e-be4ba040739b + jinja: 'Translate this into Gurajati: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_gu-en-source+target + - translate-en-gu-target reference: '' - 849b7ad4-3be7-415d-997e-a9013b2b4447: !Template + 8597ba15-219a-4d23-aea7-8262b1548226: !Template answer_choices: null - id: 849b7ad4-3be7-415d-997e-a9013b2b4447 + id: 8597ba15-219a-4d23-aea7-8262b1548226 jinja: 'If the Gurajati version says: {{translation["gu"]}}; then the English version should say: @@ -171,25 +172,26 @@ templates: name: !!python/tuple - version-gu-en-source+target reference: '' - 8cf8e194-abb9-43e6-a1e1-e8d3b9e5423f: !Template + 9a033c1a-fefc-46bc-84b7-299a7724c985: !Template answer_choices: null - id: 8cf8e194-abb9-43e6-a1e1-e8d3b9e5423f - jinja: 'If the original version says: {{translation["gu"]}}; then the English - version should say: + id: 9a033c1a-fefc-46bc-84b7-299a7724c985 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Gurajati translation is: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-gu-en-target + - a_good_translation-en-gu-source+target reference: '' - 97ceebb5-89a8-47e7-b5f8-49e070f3af8b: !Template + 9dcf2e80-7b92-4117-a713-e2e299d9cf39: !Template answer_choices: null - id: 97ceebb5-89a8-47e7-b5f8-49e070f3af8b - jinja: 'How do you say {{translation["gu"]}} in English? + id: 9dcf2e80-7b92-4117-a713-e2e299d9cf39 + jinja: 'Given the following passage: {{translation["gu"]}} , a good English translation + is: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -198,111 +200,108 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-gu-en-target + - a_good_translation-gu-en-target reference: '' - 9a5136ee-169b-45d8-ad8b-67765661da30: !Template + ae16add5-c066-40e7-a7c9-71926206d849: !Template answer_choices: null - id: 9a5136ee-169b-45d8-ad8b-67765661da30 - jinja: 'Gurajati: {{translation["gu"]}} = English: - - ||| {{translation["en"]}}' + id: ae16add5-c066-40e7-a7c9-71926206d849 + jinja: 'Given the following source text in Gurajati: {{translation["gu"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-gu-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - b973b4b8-2dd5-407f-86db-947f40168c67: !Template + - a_good_translation-gu-en-source+target + reference: '' + b3f958e4-a4d9-4ba7-91e6-8958f5612a47: !Template answer_choices: null - id: b973b4b8-2dd5-407f-86db-947f40168c67 - jinja: '{{translation["gu"]}} translates into English as: + id: b3f958e4-a4d9-4ba7-91e6-8958f5612a47 + jinja: 'What is the Gurajati translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_gu-en-target - reference: '' - d543de51-a465-4389-b155-400ac558579a: !Template + - gpt-3-en-gu-target + reference: GPT-3 paper + c8441059-2e74-4248-862b-901e053d6c36: !Template answer_choices: null - id: d543de51-a465-4389-b155-400ac558579a - jinja: 'How do you say {{translation["en"]}} in Gurajati? + id: c8441059-2e74-4248-862b-901e053d6c36 + jinja: 'How do you say {{translation["gu"]}} in English? - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-gu-target + - how_to_say-gu-en-target reference: '' - da75185f-a87d-400f-bfde-5fc6f668a91d: !Template + ccc7f258-d5ce-48d2-a833-9bfd4e42f42c: !Template answer_choices: null - id: da75185f-a87d-400f-bfde-5fc6f668a91d - jinja: ' {{translation["en"]}} translates into Gurajati as: + id: ccc7f258-d5ce-48d2-a833-9bfd4e42f42c + jinja: 'What is the English translation of the Gurajati sentence: {{translation["gu"]}} - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-gu-target - reference: '' - e4ff9670-a294-4b3f-bc32-f9779669ee16: !Template + - gpt-3-gu-en-source+target + reference: GPT-3 paper + cf3bf7ca-02f5-44a1-ab32-ff453bc05c1a: !Template answer_choices: null - id: e4ff9670-a294-4b3f-bc32-f9779669ee16 - jinja: 'Translate Gurajati into English: {{translation["gu"]}} + id: cf3bf7ca-02f5-44a1-ab32-ff453bc05c1a + jinja: '{{translation["gu"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-gu-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e64f9072-af1a-4f09-aad7-f55280518f8b: !Template + - translate_as_gu-en-target + reference: '' + cf759928-85d0-4779-ae73-e3cc64c63a08: !Template answer_choices: null - id: e64f9072-af1a-4f09-aad7-f55280518f8b - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Gurajati translation is: - - ||| {{translation["gu"]}}' + id: cf759928-85d0-4779-ae73-e3cc64c63a08 + jinja: 'Given the following passage: {{translation["en"]}} , a good Gurajati translation + is: ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-gu-source+target + - a_good_translation-en-gu-target reference: '' - e86e178f-db53-491a-aac3-c2b80a0c6f27: !Template + d4921d87-f2c4-4f7b-98ac-f330a93b672b: !Template answer_choices: null - id: e86e178f-db53-491a-aac3-c2b80a0c6f27 - jinja: 'English: {{translation["en"]}} = Gurajati: + id: d4921d87-f2c4-4f7b-98ac-f330a93b672b + jinja: 'What is the English translation of : {{translation["gu"]}} - ||| {{translation["gu"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-gu-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - ec1f81e9-8b1d-44de-a503-68b67f9e1ffa: !Template + - gpt-3-gu-en-target + reference: GPT-3 paper + e1337b9d-8f65-4ab7-bd13-e2a65cff164e: !Template answer_choices: null - id: ec1f81e9-8b1d-44de-a503-68b67f9e1ffa - jinja: '{{translation["gu"]}} = English: + id: e1337b9d-8f65-4ab7-bd13-e2a65cff164e + jinja: 'Gurajati: {{translation["gu"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -311,11 +310,11 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-gu-en-target + - xglm-gu-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - f275a19c-3927-4073-8abd-d39d97f3f70f: !Template + e85a53f5-a00d-4a2d-b6b3-3ae64230d6fd: !Template answer_choices: null - id: f275a19c-3927-4073-8abd-d39d97f3f70f + id: e85a53f5-a00d-4a2d-b6b3-3ae64230d6fd jinja: '{{translation["en"]}} = Gurajati: ||| {{translation["gu"]}}' @@ -327,24 +326,25 @@ templates: name: !!python/tuple - xglm-en-gu-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - f43e4d43-b409-48ef-8e16-1c21a05f8369: !Template + e860b831-fed6-439e-bafa-b9fc04fc8873: !Template answer_choices: null - id: f43e4d43-b409-48ef-8e16-1c21a05f8369 - jinja: 'What is the English translation of : {{translation["gu"]}} + id: e860b831-fed6-439e-bafa-b9fc04fc8873 + jinja: 'If the English version says: {{translation["en"]}}; then the Gurajati + version should say: - ||| {{translation["en"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-gu-en-target - reference: GPT-3 paper - f56533a5-e686-4ff1-8821-08d3482af548: !Template + - version-en-gu-source+target + reference: '' + ec9cae8d-c7ee-4e41-bc44-faa289de9a74: !Template answer_choices: null - id: f56533a5-e686-4ff1-8821-08d3482af548 - jinja: 'What is the English translation of the Gurajati sentence: {{translation["gu"]}} + id: ec9cae8d-c7ee-4e41-bc44-faa289de9a74 + jinja: 'Translate this from Gurajati into English: {{translation["gu"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -353,19 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-gu-en-source+target - reference: GPT-3 paper - f6d40b63-a53d-4f09-9d57-22e41a862baa: !Template + - translate-gu-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + f5a969a6-5dca-47b1-b15a-6cb4d296cb1d: !Template answer_choices: null - id: f6d40b63-a53d-4f09-9d57-22e41a862baa - jinja: 'Translate English into Gurajati: {{translation["en"]}} + id: f5a969a6-5dca-47b1-b15a-6cb4d296cb1d + jinja: ' {{translation["en"]}} translates into Gurajati as: - ||| {{translation["gu"]}}' + ||| {{translation["gu"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-gu-source+starget + - translate_as_en-gu-target reference: '' diff --git a/promptsource/templates/wmt19/kk-en/templates.yaml b/promptsource/templates/wmt19/kk-en/templates.yaml index 9c7076d6a..91a05e502 100644 --- a/promptsource/templates/wmt19/kk-en/templates.yaml +++ b/promptsource/templates/wmt19/kk-en/templates.yaml @@ -1,80 +1,66 @@ dataset: wmt19 subset: kk-en templates: - 19bc7d44-4965-40b8-9198-653b1766871c: !Template + 016e1027-677c-4fff-9f87-4f0b05ff7f98: !Template answer_choices: null - id: 19bc7d44-4965-40b8-9198-653b1766871c - jinja: '{{translation["kk"]}} translates into English as: + id: 016e1027-677c-4fff-9f87-4f0b05ff7f98 + jinja: 'What is the Kazakh translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_kk-en-target - reference: '' - 1d10e5f9-ae18-42c1-a752-d3d4543e3bc5: !Template + - gpt-3-en-kk-target + reference: GPT-3 paper + 05b7fa2e-0a0e-4e7f-8fa7-71aa04f40dc6: !Template answer_choices: null - id: 1d10e5f9-ae18-42c1-a752-d3d4543e3bc5 - jinja: 'If the English version says: {{translation["en"]}}; then the Kazakh version - should say: + id: 05b7fa2e-0a0e-4e7f-8fa7-71aa04f40dc6 + jinja: 'How do you say {{translation["en"]}} in Kazakh? - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-kk-source+target + - how_to_say-en-kk-target reference: '' - 1d87080c-c1b6-49a7-aced-a6aa96da7f7c: !Template - answer_choices: null - id: 1d87080c-c1b6-49a7-aced-a6aa96da7f7c - jinja: 'Kazakh: {{translation["kk"]}} = English: - - ||| {{translation["en"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - xglm-kk-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 1f5d3cf4-76cf-4629-875e-e9730ef17594: !Template + 05d095c2-f150-46e3-a39b-1f06e01ca39e: !Template answer_choices: null - id: 1f5d3cf4-76cf-4629-875e-e9730ef17594 - jinja: 'English: {{translation["en"]}} translates into Kazakh as: + id: 05d095c2-f150-46e3-a39b-1f06e01ca39e + jinja: 'If the original version says: {{translation["en"]}}; then the Kazakh version + should say: - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-kk-source+target + - version-en-kk-target reference: '' - 26a42802-9354-4fce-9f6b-e44819534b76: !Template + 12139536-fb0e-43b5-a763-d9a6c84101ba: !Template answer_choices: null - id: 26a42802-9354-4fce-9f6b-e44819534b76 - jinja: 'What is the English translation of the Kazakh sentence: {{translation["kk"]}} + id: 12139536-fb0e-43b5-a763-d9a6c84101ba + jinja: 'Kazakh: {{translation["kk"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-kk-en-source+target - reference: GPT-3 paper - 56b3c419-8774-4c38-aae0-969f3ce948cb: !Template + - xglm-kk-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 29f05e00-9b0f-4db0-9226-f558fd02709d: !Template answer_choices: null - id: 56b3c419-8774-4c38-aae0-969f3ce948cb + id: 29f05e00-9b0f-4db0-9226-f558fd02709d jinja: 'If the Kazakh version says: {{translation["kk"]}}; then the English version should say: @@ -87,10 +73,10 @@ templates: name: !!python/tuple - version-kk-en-source+target reference: '' - 5c70e3bc-ae9d-448c-951b-067bc459e666: !Template + 2f05a859-a361-4e6a-ac89-48c2b66f0382: !Template answer_choices: null - id: 5c70e3bc-ae9d-448c-951b-067bc459e666 - jinja: 'Translate into Kazakh: {{translation["en"]}} + id: 2f05a859-a361-4e6a-ac89-48c2b66f0382 + jinja: 'Translate this from English into Kazakh: {{translation["en"]}} ||| {{translation["kk"]}}' metadata: !TemplateMetadata @@ -99,140 +85,152 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-kk-target + - translate-en-kk-source+starget reference: '' - 65e01c0f-1b92-490a-a0e3-95f8d65c0897: !Template + 35258de4-8943-4a5f-b45c-72559fc048bc: !Template answer_choices: null - id: 65e01c0f-1b92-490a-a0e3-95f8d65c0897 - jinja: 'Translate English into Kazakh: {{translation["en"]}} + id: 35258de4-8943-4a5f-b45c-72559fc048bc + jinja: ' {{translation["en"]}} translates into Kazakh as: - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-kk-source+starget + - translate_as_en-kk-target reference: '' - 6a379888-9584-4fde-beb6-ac021d7f3927: !Template + 361734f8-dc83-4194-87ad-937a210cc037: !Template answer_choices: null - id: 6a379888-9584-4fde-beb6-ac021d7f3927 - jinja: 'If the original version says: {{translation["en"]}}; then the Kazakh version - should say: + id: 361734f8-dc83-4194-87ad-937a210cc037 + jinja: 'What is the English translation of : {{translation["kk"]}} - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-kk-target - reference: '' - 6a88262a-bb53-489c-b6d9-7439a5db4dd9: !Template + - gpt-3-kk-en-target + reference: GPT-3 paper + 3991795b-1860-4ffa-9001-26454c7174b2: !Template answer_choices: null - id: 6a88262a-bb53-489c-b6d9-7439a5db4dd9 - jinja: 'What is the English translation of : {{translation["kk"]}} + id: 3991795b-1860-4ffa-9001-26454c7174b2 + jinja: 'English: {{translation["en"]}} = Kazakh: - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-kk-en-target - reference: GPT-3 paper - 6c0c700f-76c2-4673-ad31-a24bde943eb1: !Template + - xglm-en-kk-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 45807ca5-c5df-4a43-a7f1-3935d49dbbe7: !Template answer_choices: null - id: 6c0c700f-76c2-4673-ad31-a24bde943eb1 - jinja: 'How do you say {{translation["en"]}} in Kazakh? + id: 45807ca5-c5df-4a43-a7f1-3935d49dbbe7 + jinja: 'If the original version says: {{translation["kk"]}}; then the English + version should say: - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-kk-target + - version-kk-en-target reference: '' - 724de4bc-efb7-4203-a82d-983a38d2802e: !Template + 48ac727e-ecd6-4458-a8e2-9c1e1e9d1f73: !Template answer_choices: null - id: 724de4bc-efb7-4203-a82d-983a38d2802e - jinja: 'Translate into English: {{translation["kk"]}} + id: 48ac727e-ecd6-4458-a8e2-9c1e1e9d1f73 + jinja: 'If the English version says: {{translation["en"]}}; then the Kazakh version + should say: - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-kk-en-target + - version-en-kk-source+target reference: '' - 7dd8375e-4100-4030-92c0-8fdd4b2d04ba: !Template + 5abf7d89-6eca-4d84-809a-1897af755a42: !Template answer_choices: null - id: 7dd8375e-4100-4030-92c0-8fdd4b2d04ba - jinja: 'Translate Kazakh into English: {{translation["kk"]}} - - ||| {{translation["en"]}}' + id: 5abf7d89-6eca-4d84-809a-1897af755a42 + jinja: 'Given the following source text in Kazakh: {{translation["kk"]}} , a good + English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-kk-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 88171545-f09a-40d1-b9cc-4e391a23f0f8: !Template + - a_good_translation-kk-en-source+target + reference: '' + 6b67073b-86ff-41f5-ace3-87380619aa1a: !Template answer_choices: null - id: 88171545-f09a-40d1-b9cc-4e391a23f0f8 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Kazakh translation is: + id: 6b67073b-86ff-41f5-ace3-87380619aa1a + jinja: 'Translate this into Kazakh: {{translation["en"]}} - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-kk-source+target + - translate-en-kk-target reference: '' - a1638c3a-8190-4a18-8384-4b0d20cf975e: !Template + 7a00cb3d-d7e9-40e0-b2f1-a639caef4105: !Template answer_choices: null - id: a1638c3a-8190-4a18-8384-4b0d20cf975e - jinja: 'If the original version says: {{translation["kk"]}}; then the English - version should say: + id: 7a00cb3d-d7e9-40e0-b2f1-a639caef4105 + jinja: '{{translation["kk"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-kk-en-target + - translate_as_kk-en-target reference: '' - b52dee88-79ee-4a7c-876e-3f71764a8aee: !Template + 7e23e55c-1e67-4a21-98e3-a361eb989e98: !Template answer_choices: null - id: b52dee88-79ee-4a7c-876e-3f71764a8aee - jinja: 'Given the following source text in Kazakh: {{translation["kk"]}} , a good - English translation is: ||| {{translation["en"]}}' + id: 7e23e55c-1e67-4a21-98e3-a361eb989e98 + jinja: '{{translation["kk"]}} = English: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-kk-en-source+target + - xglm-kk-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 92d739d1-ad1c-4379-bd0f-6d2584bb6147: !Template + answer_choices: null + id: 92d739d1-ad1c-4379-bd0f-6d2584bb6147 + jinja: 'Given the following passage: {{translation["en"]}} , a good Kazakh translation + is: ||| {{translation["kk"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-en-kk-target reference: '' - b56ca813-ce38-44c9-86e2-160e22f4dc1e: !Template + a2528e1c-b6e5-4c2d-95ce-395d6d5dccc7: !Template answer_choices: null - id: b56ca813-ce38-44c9-86e2-160e22f4dc1e - jinja: 'How do you say {{translation["kk"]}} in English? + id: a2528e1c-b6e5-4c2d-95ce-395d6d5dccc7 + jinja: 'Kazakh: {{translation["kk"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -241,39 +239,41 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-kk-en-target + - translate_as_kk-en-source+target reference: '' - c56d3846-cc9b-41a0-aa4f-a1beedd65ec2: !Template + ac7eb2a0-7979-4045-bd86-65501805c2e0: !Template answer_choices: null - id: c56d3846-cc9b-41a0-aa4f-a1beedd65ec2 - jinja: 'English: {{translation["en"]}} = Kazakh: + id: ac7eb2a0-7979-4045-bd86-65501805c2e0 + jinja: 'Translate this from Kazakh into English: {{translation["kk"]}} - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-kk-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - db0dd18d-e620-442d-9dbd-10bfebd1d433: !Template + - translate-kk-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + bdac0d10-e077-4c76-9f7e-f6b6878e41ad: !Template answer_choices: null - id: db0dd18d-e620-442d-9dbd-10bfebd1d433 - jinja: 'Given the following passage: {{translation["en"]}} , a good Kazakh translation - is: ||| {{translation["kk"]}}' + id: bdac0d10-e077-4c76-9f7e-f6b6878e41ad + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Kazakh translation is: + + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-kk-target + - a_good_translation-en-kk-source+target reference: '' - e2896e24-9b0b-466d-989f-f0ed6ab8842a: !Template + c03acc66-b187-4746-b5fe-4b14ab4ef417: !Template answer_choices: null - id: e2896e24-9b0b-466d-989f-f0ed6ab8842a - jinja: ' {{translation["en"]}} translates into Kazakh as: + id: c03acc66-b187-4746-b5fe-4b14ab4ef417 + jinja: 'English: {{translation["en"]}} translates into Kazakh as: ||| {{translation["kk"]}}' metadata: !TemplateMetadata @@ -282,54 +282,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-kk-target + - translate_as_en-kk-source+target reference: '' - e31ce0bc-2ac2-46de-8254-d9c3ed665c18: !Template + c7bc8a8c-829e-4863-a978-0ac6aee572ff: !Template answer_choices: null - id: e31ce0bc-2ac2-46de-8254-d9c3ed665c18 - jinja: 'What is the Kazakh translation of: {{translation["en"]}} + id: c7bc8a8c-829e-4863-a978-0ac6aee572ff + jinja: 'Translate this into English: {{translation["kk"]}} - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-kk-target - reference: GPT-3 paper - e3d7e25f-3fc1-44ec-8974-1c89d7976e15: !Template + - translate-kk-en-target + reference: '' + d263f205-a7e9-450f-a821-6cb5dd167585: !Template answer_choices: null - id: e3d7e25f-3fc1-44ec-8974-1c89d7976e15 - jinja: '{{translation["kk"]}} = English: + id: d263f205-a7e9-450f-a821-6cb5dd167585 + jinja: '{{translation["en"]}} = Kazakh: - ||| {{translation["en"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-kk-en-target + - xglm-en-kk-target reference: XGLM paper https://arxiv.org/abs/2112.10668 - e568db81-1c82-4e2f-bd74-d86a67ce4fc9: !Template + e59ffaa7-d828-463f-b79d-6fe0e0378daa: !Template answer_choices: null - id: e568db81-1c82-4e2f-bd74-d86a67ce4fc9 - jinja: 'What is the Kazakh translation of the English sentence: {{translation["en"]}} + id: e59ffaa7-d828-463f-b79d-6fe0e0378daa + jinja: 'Given the following passage: {{translation["kk"]}} , a good English translation + is: - ||| {{translation["kk"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-kk-source+target - reference: GPT 3 paper - f0cb7953-a0a6-4f13-8023-bbb2dffe259e: !Template + - a_good_translation-kk-en-target + reference: '' + e614b3f0-ac5f-4570-ab95-8f246be0f077: !Template answer_choices: null - id: f0cb7953-a0a6-4f13-8023-bbb2dffe259e - jinja: 'Kazakh: {{translation["kk"]}} translates into English as: + id: e614b3f0-ac5f-4570-ab95-8f246be0f077 + jinja: 'How do you say {{translation["kk"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -338,34 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_kk-en-source+target + - how_to_say-kk-en-target reference: '' - f2c0559c-5768-432a-b5dc-b9aba5577038: !Template + f617d102-ce28-4b31-aae9-5ffd6341a8ba: !Template answer_choices: null - id: f2c0559c-5768-432a-b5dc-b9aba5577038 - jinja: '{{translation["en"]}} = Kazakh: + id: f617d102-ce28-4b31-aae9-5ffd6341a8ba + jinja: 'What is the Kazakh translation of the English sentence: {{translation["en"]}} - ||| {{translation["kk"]}}' + ||| {{translation["kk"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-kk-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f7171f67-3c41-44f8-8f75-c5958ea7811f: !Template + - gpt-3-en-kk-source+target + reference: GPT 3 paper + ff749f28-c9d2-4830-a8bf-c7895e00aaaa: !Template answer_choices: null - id: f7171f67-3c41-44f8-8f75-c5958ea7811f - jinja: 'Given the following passage: {{translation["kk"]}} , a good English translation - is: + id: ff749f28-c9d2-4830-a8bf-c7895e00aaaa + jinja: 'What is the English translation of the Kazakh sentence: {{translation["kk"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-kk-en-target - reference: '' + - gpt-3-kk-en-source+target + reference: GPT-3 paper diff --git a/promptsource/templates/wmt19/lt-en/templates.yaml b/promptsource/templates/wmt19/lt-en/templates.yaml index b28e3fa50..c127ea6f0 100644 --- a/promptsource/templates/wmt19/lt-en/templates.yaml +++ b/promptsource/templates/wmt19/lt-en/templates.yaml @@ -1,38 +1,38 @@ dataset: wmt19 subset: lt-en templates: - 09d912d9-54b5-4094-886a-78e9d3622c49: !Template + 005f744f-cd93-4d4b-8729-56bc2188669d: !Template answer_choices: null - id: 09d912d9-54b5-4094-886a-78e9d3622c49 - jinja: 'Given the following passage: {{translation["en"]}} , a good Lithuanian - translation is: ||| {{translation["lt"]}}' + id: 005f744f-cd93-4d4b-8729-56bc2188669d + jinja: 'Translate this from Lithuanian into English: {{translation["lt"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lt-target - reference: '' - 1081536f-f4d4-47b5-9b08-e75756182e0e: !Template + - translate-lt-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 0380efec-57ed-4437-84dc-a61f79017856: !Template answer_choices: null - id: 1081536f-f4d4-47b5-9b08-e75756182e0e - jinja: 'If the Lithuanian version says: {{translation["lt"]}}; then the English - version should say: + id: 0380efec-57ed-4437-84dc-a61f79017856 + jinja: 'Lithuanian: {{translation["lt"]}} translates into English as: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-lt-en-source+target + - translate_as_lt-en-source+target reference: '' - 17bcfc14-24c3-410e-861a-9758751339f1: !Template + 0ac8a6cc-615a-4b37-864f-66a223c5dbd9: !Template answer_choices: null - id: 17bcfc14-24c3-410e-861a-9758751339f1 - jinja: 'Lithuanian: {{translation["lt"]}} translates into English as: + id: 0ac8a6cc-615a-4b37-864f-66a223c5dbd9 + jinja: 'Lithuanian: {{translation["lt"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -41,54 +41,55 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_lt-en-source+target - reference: '' - 19f026f2-7bf7-48e7-93f8-d711e746bbbd: !Template + - xglm-lt-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 18c81ecf-7582-49a8-a4c8-5bb4281ce98f: !Template answer_choices: null - id: 19f026f2-7bf7-48e7-93f8-d711e746bbbd - jinja: 'English: {{translation["en"]}} translates into Lithuanian as: + id: 18c81ecf-7582-49a8-a4c8-5bb4281ce98f + jinja: 'What is the Lithuanian translation of the English sentence: {{translation["en"]}} - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lt-source+target - reference: '' - 1d31906c-be02-47a3-926f-0bd9c4bf05c9: !Template + - gpt-3-en-lt-source+target + reference: GPT 3 paper + 1a6f9056-57b3-41ab-a365-a7932bee428f: !Template answer_choices: null - id: 1d31906c-be02-47a3-926f-0bd9c4bf05c9 - jinja: 'What is the English translation of the Lithuanian sentence: {{translation["lt"]}} + id: 1a6f9056-57b3-41ab-a365-a7932bee428f + jinja: 'How do you say {{translation["lt"]}} in English? - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lt-en-source+target - reference: GPT-3 paper - 239dec95-271d-4301-8cab-fca10958c97f: !Template + - how_to_say-lt-en-target + reference: '' + 1ce4b380-5230-41f7-b8bb-824cacdf80ee: !Template answer_choices: null - id: 239dec95-271d-4301-8cab-fca10958c97f - jinja: ' {{translation["en"]}} translates into Lithuanian as: + id: 1ce4b380-5230-41f7-b8bb-824cacdf80ee + jinja: 'If the original version says: {{translation["en"]}}; then the Lithuanian + version should say: - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-lt-target + - version-en-lt-target reference: '' - 2486e08b-e932-4f7c-b2fe-a5ff88e4a5a4: !Template + 264b2535-2e7d-4efe-9704-09c2b1332cac: !Template answer_choices: null - id: 2486e08b-e932-4f7c-b2fe-a5ff88e4a5a4 - jinja: 'Translate into Lithuanian: {{translation["en"]}} + id: 264b2535-2e7d-4efe-9704-09c2b1332cac + jinja: 'Translate this into Lithuanian: {{translation["en"]}} ||| {{translation["lt"]}}' metadata: !TemplateMetadata @@ -99,24 +100,23 @@ templates: name: !!python/tuple - translate-en-lt-target reference: '' - 388c481e-9ed2-4860-87b0-00a32b3f5a4e: !Template + 28987179-bc04-4e63-982a-b959efbd95a5: !Template answer_choices: null - id: 388c481e-9ed2-4860-87b0-00a32b3f5a4e - jinja: '{{translation["en"]}} = Lithuanian: - - ||| {{translation["lt"]}}' + id: 28987179-bc04-4e63-982a-b959efbd95a5 + jinja: 'Given the following passage: {{translation["en"]}} , a good Lithuanian + translation is: ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-lt-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 3cea9425-dbaa-4c4c-a1b0-133683bcca53: !Template + - a_good_translation-en-lt-target + reference: '' + 4bb2c423-342a-46fc-928b-eb4d9a01297c: !Template answer_choices: null - id: 3cea9425-dbaa-4c4c-a1b0-133683bcca53 - jinja: '{{translation["lt"]}} translates into English as: + id: 4bb2c423-342a-46fc-928b-eb4d9a01297c + jinja: '{{translation["lt"]}} = English: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -125,182 +125,182 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_lt-en-target - reference: '' - 48ed0004-1cf3-42e2-854c-756a73d99eb9: !Template + - xglm-lt-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 4f5f0594-32a5-4690-81ae-64deb3289f94: !Template answer_choices: null - id: 48ed0004-1cf3-42e2-854c-756a73d99eb9 - jinja: 'If the original version says: {{translation["en"]}}; then the Lithuanian - version should say: + id: 4f5f0594-32a5-4690-81ae-64deb3289f94 + jinja: 'Translate this into English: {{translation["lt"]}} - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lt-target + - translate-lt-en-target reference: '' - 604d7587-c23e-443e-ab71-4c06e55f1463: !Template + 5afbe8cf-9f27-4482-b936-44a06fae2a21: !Template answer_choices: null - id: 604d7587-c23e-443e-ab71-4c06e55f1463 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Lithuanian translation is: + id: 5afbe8cf-9f27-4482-b936-44a06fae2a21 + jinja: 'If the Lithuanian version says: {{translation["lt"]}}; then the English + version should say: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-lt-source+target + - version-lt-en-source+target reference: '' - 7006b213-cc36-4153-ae89-b85a73ec598e: !Template + 615c0d2b-ead7-458a-b3b8-16092d899d0f: !Template answer_choices: null - id: 7006b213-cc36-4153-ae89-b85a73ec598e - jinja: 'What is the English translation of : {{translation["lt"]}} + id: 615c0d2b-ead7-458a-b3b8-16092d899d0f + jinja: 'If the English version says: {{translation["en"]}}; then the Lithuanian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-lt-en-target - reference: GPT-3 paper - 79d4fccc-593b-4dce-b6ef-3b3c6c189de4: !Template + - version-en-lt-source+target + reference: '' + 74fb4a3d-99a1-4bd0-a57d-2bd53732eb35: !Template answer_choices: null - id: 79d4fccc-593b-4dce-b6ef-3b3c6c189de4 - jinja: 'Given the following source text in Lithuanian: {{translation["lt"]}} , - a good English translation is: ||| {{translation["en"]}}' + id: 74fb4a3d-99a1-4bd0-a57d-2bd53732eb35 + jinja: 'English: {{translation["en"]}} = Lithuanian: + + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lt-en-source+target - reference: '' - 8ba668c5-9ad8-4718-aa54-2e5d8b746a73: !Template + - xglm-en-lt-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 77a3ab9f-f3ce-4b46-b080-13776b973919: !Template answer_choices: null - id: 8ba668c5-9ad8-4718-aa54-2e5d8b746a73 - jinja: '{{translation["lt"]}} = English: + id: 77a3ab9f-f3ce-4b46-b080-13776b973919 + jinja: 'How do you say {{translation["en"]}} in Lithuanian? - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-lt-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 9d160f00-e1d9-477c-b71a-4ded861367f9: !Template + - how_to_say-en-lt-target + reference: '' + 8d69a032-f7c1-49ff-acea-093b672c9dd6: !Template answer_choices: null - id: 9d160f00-e1d9-477c-b71a-4ded861367f9 - jinja: 'Lithuanian: {{translation["lt"]}} = English: + id: 8d69a032-f7c1-49ff-acea-093b672c9dd6 + jinja: 'What is the Lithuanian translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-lt-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c342a4ea-e09b-445c-96f5-c79341d5cb40: !Template + - gpt-3-en-lt-target + reference: GPT-3 paper + 97a087e8-fd66-4b95-9c75-87250b8e5e5c: !Template answer_choices: null - id: c342a4ea-e09b-445c-96f5-c79341d5cb40 - jinja: 'How do you say {{translation["en"]}} in Lithuanian? + id: 97a087e8-fd66-4b95-9c75-87250b8e5e5c + jinja: 'Given the following passage: {{translation["lt"]}} , a good English translation + is: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-lt-target + - a_good_translation-lt-en-target reference: '' - c978e375-5b34-45a8-87aa-22b51cccb868: !Template + a09584a3-544a-4508-9890-7db646ec6902: !Template answer_choices: null - id: c978e375-5b34-45a8-87aa-22b51cccb868 - jinja: 'What is the Lithuanian translation of the English sentence: {{translation["en"]}} + id: a09584a3-544a-4508-9890-7db646ec6902 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Lithuanian translation is: - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lt-source+target - reference: GPT 3 paper - cbc63670-78c7-45d1-9ceb-451b9c22f31d: !Template + - a_good_translation-en-lt-source+target + reference: '' + b4b2863d-0eb4-4220-86ba-cd7873a31e33: !Template answer_choices: null - id: cbc63670-78c7-45d1-9ceb-451b9c22f31d - jinja: 'Translate into English: {{translation["lt"]}} + id: b4b2863d-0eb4-4220-86ba-cd7873a31e33 + jinja: 'English: {{translation["en"]}} translates into Lithuanian as: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lt-en-target + - translate_as_en-lt-source+target reference: '' - cf64d506-a054-49cf-83f9-57eb3923680e: !Template + ba8c242b-aeac-44b0-8ed0-4e108fa46ddc: !Template answer_choices: null - id: cf64d506-a054-49cf-83f9-57eb3923680e - jinja: 'If the English version says: {{translation["en"]}}; then the Lithuanian - version should say: + id: ba8c242b-aeac-44b0-8ed0-4e108fa46ddc + jinja: 'What is the English translation of the Lithuanian sentence: {{translation["lt"]}} - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-lt-source+target - reference: '' - d1951cc8-cef9-49ab-9df1-a9a58caa38fc: !Template + - gpt-3-lt-en-source+target + reference: GPT-3 paper + bb388f4f-0184-4ee1-b445-6ce35a4566a4: !Template answer_choices: null - id: d1951cc8-cef9-49ab-9df1-a9a58caa38fc - jinja: 'What is the Lithuanian translation of: {{translation["en"]}} + id: bb388f4f-0184-4ee1-b445-6ce35a4566a4 + jinja: 'What is the English translation of : {{translation["lt"]}} - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-lt-target + - gpt-3-lt-en-target reference: GPT-3 paper - dd1e7adb-8331-4a0d-844e-91076648f25a: !Template + c2fb9339-44a0-468b-aa07-984aaf09cb9c: !Template answer_choices: null - id: dd1e7adb-8331-4a0d-844e-91076648f25a - jinja: 'Given the following passage: {{translation["lt"]}} , a good English translation - is: - - ||| {{translation["en"]}}' + id: c2fb9339-44a0-468b-aa07-984aaf09cb9c + jinja: 'Given the following source text in Lithuanian: {{translation["lt"]}} , + a good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-lt-en-target + - a_good_translation-lt-en-source+target reference: '' - de291c64-3989-4104-8c50-655c2fa73969: !Template + c8412be4-f223-47d1-ad3f-36c640edaaaa: !Template answer_choices: null - id: de291c64-3989-4104-8c50-655c2fa73969 + id: c8412be4-f223-47d1-ad3f-36c640edaaaa jinja: 'If the original version says: {{translation["lt"]}}; then the English version should say: @@ -313,59 +313,59 @@ templates: name: !!python/tuple - version-lt-en-target reference: '' - dec5fe1c-5d8a-477d-8a63-cdbf1d0374f1: !Template + cab29758-8001-4d7f-9eaa-b492cb0aca49: !Template answer_choices: null - id: dec5fe1c-5d8a-477d-8a63-cdbf1d0374f1 - jinja: 'Translate English into Lithuanian: {{translation["en"]}} + id: cab29758-8001-4d7f-9eaa-b492cb0aca49 + jinja: '{{translation["lt"]}} translates into English as: - ||| {{translation["lt"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-lt-source+starget + - translate_as_lt-en-target reference: '' - ea94c785-cff2-47f5-a06d-f31ce7885b7f: !Template + da751307-9d0b-4566-bb95-9fa90f0132f5: !Template answer_choices: null - id: ea94c785-cff2-47f5-a06d-f31ce7885b7f - jinja: 'How do you say {{translation["lt"]}} in English? + id: da751307-9d0b-4566-bb95-9fa90f0132f5 + jinja: ' {{translation["en"]}} translates into Lithuanian as: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - how_to_say-lt-en-target + - translate_as_en-lt-target reference: '' - f777af07-abeb-48bb-ad9d-16f490dd1039: !Template + e5b2d433-eff7-4ca6-b10c-5cc3b3c5d662: !Template answer_choices: null - id: f777af07-abeb-48bb-ad9d-16f490dd1039 - jinja: 'Translate Lithuanian into English: {{translation["lt"]}} + id: e5b2d433-eff7-4ca6-b10c-5cc3b3c5d662 + jinja: '{{translation["en"]}} = Lithuanian: - ||| {{translation["en"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-lt-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - fb7d19ad-16ac-42bf-93a8-3960652f98c9: !Template + - xglm-en-lt-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + f3fbfb49-72a6-4aa6-b48e-f3adf02e54d5: !Template answer_choices: null - id: fb7d19ad-16ac-42bf-93a8-3960652f98c9 - jinja: 'English: {{translation["en"]}} = Lithuanian: + id: f3fbfb49-72a6-4aa6-b48e-f3adf02e54d5 + jinja: 'Translate this from English into Lithuanian: {{translation["en"]}} - ||| {{translation["lt"]}}' + ||| {{translation["lt"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-lt-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + - translate-en-lt-source+starget + reference: '' diff --git a/promptsource/templates/wmt19/ru-en/templates.yaml b/promptsource/templates/wmt19/ru-en/templates.yaml index a84c7ddad..9c1247d7a 100644 --- a/promptsource/templates/wmt19/ru-en/templates.yaml +++ b/promptsource/templates/wmt19/ru-en/templates.yaml @@ -1,25 +1,25 @@ dataset: wmt19 subset: ru-en templates: - 077da5bd-4b32-44ed-ac04-d0530e1a2364: !Template + 0aaba792-7e4c-4088-832e-8f726fb2d8f3: !Template answer_choices: null - id: 077da5bd-4b32-44ed-ac04-d0530e1a2364 - jinja: 'Given the following source text in English: {{translation["en"]}} , a - good Russian translation is: + id: 0aaba792-7e4c-4088-832e-8f726fb2d8f3 + jinja: 'If the English version says: {{translation["en"]}}; then the Russian version + should say: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-source+target + - version-en-ru-source+target reference: '' - 09c29b29-61f1-43a1-9ea2-67c2c19fcbe3: !Template + 1975db74-601b-40e5-8615-f1af51e11857: !Template answer_choices: null - id: 09c29b29-61f1-43a1-9ea2-67c2c19fcbe3 - jinja: ' {{translation["en"]}} translates into Russian as: + id: 1975db74-601b-40e5-8615-f1af51e11857 + jinja: 'How do you say {{translation["en"]}} in Russian? ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -28,95 +28,82 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_en-ru-target + - how_to_say-en-ru-target reference: '' - 0d95df60-a164-4811-b368-53b1d6c3b164: !Template + 1b9be0af-e67d-4f87-b032-172faa9e329b: !Template answer_choices: null - id: 0d95df60-a164-4811-b368-53b1d6c3b164 - jinja: '{{translation["ru"]}} = English: + id: 1b9be0af-e67d-4f87-b032-172faa9e329b + jinja: 'What is the English translation of : {{translation["ru"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-ru-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 17c15ac0-5839-491a-bcf3-156077363824: !Template + - gpt-3-ru-en-target + reference: GPT-3 paper + 2124a999-8d8e-4ba5-8eab-5e475aff39cc: !Template answer_choices: null - id: 17c15ac0-5839-491a-bcf3-156077363824 - jinja: 'English: {{translation["en"]}} translates into Russian as: + id: 2124a999-8d8e-4ba5-8eab-5e475aff39cc + jinja: 'Translate this into Russian: {{translation["en"]}} - ||| {{translation["ru"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target - reference: '' - 19d521d8-dfdb-494e-b91a-98193d6e2739: !Template - answer_choices: null - id: 19d521d8-dfdb-494e-b91a-98193d6e2739 - jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a - good English translation is: ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-ru-en-source+target + - translate-en-ru-target reference: '' - 3043dadc-4d0d-4b75-962d-5de834da539d: !Template + 256e0bdd-c8f8-4561-b6f6-e72c7acaa9c6: !Template answer_choices: null - id: 3043dadc-4d0d-4b75-962d-5de834da539d - jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} + id: 256e0bdd-c8f8-4561-b6f6-e72c7acaa9c6 + jinja: 'Given the following source text in English: {{translation["en"]}} , a + good Russian translation is: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-source+target - reference: GPT-3 paper - 315a0c39-2d00-4b04-822d-477c5cf09164: !Template + - a_good_translation-en-ru-source+target + reference: '' + 2c9a43f2-6aa1-438d-b6eb-6863ff34cc76: !Template answer_choices: null - id: 315a0c39-2d00-4b04-822d-477c5cf09164 - jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} + id: 2c9a43f2-6aa1-438d-b6eb-6863ff34cc76 + jinja: ' {{translation["en"]}} translates into Russian as: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-ru-source+target - reference: GPT 3 paper - 3b841603-c2b6-4b1f-8f79-93426cf56bc3: !Template + - translate_as_en-ru-target + reference: '' + 314b2f22-b592-4adb-8a41-2e88c5ab9a3e: !Template answer_choices: null - id: 3b841603-c2b6-4b1f-8f79-93426cf56bc3 - jinja: 'If the English version says: {{translation["en"]}}; then the Russian version - should say: + id: 314b2f22-b592-4adb-8a41-2e88c5ab9a3e + jinja: '{{translation["en"]}} = Russian: - ||| {{translation["ru"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-source+target - reference: '' - 3b8e645f-0658-4cc5-8e31-1dd43a90d2da: !Template + - xglm-en-ru-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 32681ba6-ad58-403c-b758-3d3793477327: !Template answer_choices: null - id: 3b8e645f-0658-4cc5-8e31-1dd43a90d2da + id: 32681ba6-ad58-403c-b758-3d3793477327 jinja: 'What is the Russian translation of: {{translation["en"]}} ||| {{translation["ru"]}}' @@ -128,23 +115,24 @@ templates: name: !!python/tuple - gpt-3-en-ru-target reference: GPT-3 paper - 4c399c8b-52e3-4720-9aca-aba5f7345148: !Template + 3cc7638d-91cc-4838-a4a4-beb612ee2051: !Template answer_choices: null - id: 4c399c8b-52e3-4720-9aca-aba5f7345148 - jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation - is: ||| {{translation["ru"]}}' + id: 3cc7638d-91cc-4838-a4a4-beb612ee2051 + jinja: 'Translate this from Russian into English: {{translation["ru"]}} + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-ru-target - reference: '' - 551f97bf-1d7a-4400-bcb6-f79a4c9c3a8f: !Template + - translate-ru-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 3e7069c6-6efb-4865-b18e-e67bd4425c5c: !Template answer_choices: null - id: 551f97bf-1d7a-4400-bcb6-f79a4c9c3a8f - jinja: 'How do you say {{translation["en"]}} in Russian? + id: 3e7069c6-6efb-4865-b18e-e67bd4425c5c + jinja: 'English: {{translation["en"]}} translates into Russian as: ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -153,12 +141,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-ru-target + - translate_as_en-ru-source+target reference: '' - 66fbd36a-710b-4277-8859-2cdf7fc75982: !Template + 46b88a9c-4e5e-4479-9840-85454dbcb303: !Template answer_choices: null - id: 66fbd36a-710b-4277-8859-2cdf7fc75982 - jinja: 'Russian: {{translation["ru"]}} translates into English as: + id: 46b88a9c-4e5e-4479-9840-85454dbcb303 + jinja: '{{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -167,27 +155,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-source+target + - translate_as_ru-en-target reference: '' - 78c33a2d-6895-491a-abce-ec474193048d: !Template + 4861eaf1-a385-4b2c-97e7-47fd0f484973: !Template answer_choices: null - id: 78c33a2d-6895-491a-abce-ec474193048d - jinja: 'If the original version says: {{translation["en"]}}; then the Russian - version should say: + id: 4861eaf1-a385-4b2c-97e7-47fd0f484973 + jinja: 'Translate this into English: {{translation["ru"]}} - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-ru-target + - translate-ru-en-target reference: '' - 81e99ccc-91f9-45e2-94c4-add17ed68ef4: !Template + 56a6ef90-d916-4872-9971-28d32142af96: !Template answer_choices: null - id: 81e99ccc-91f9-45e2-94c4-add17ed68ef4 - jinja: 'Translate into English: {{translation["ru"]}} + id: 56a6ef90-d916-4872-9971-28d32142af96 + jinja: 'What is the English translation of the Russian sentence: {{translation["ru"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -196,12 +183,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-target - reference: '' - 8934e560-7bed-4718-8864-4356aae19c72: !Template + - gpt-3-ru-en-source+target + reference: GPT-3 paper + 5e34c936-eb95-49fd-abf8-76e069250e6a: !Template answer_choices: null - id: 8934e560-7bed-4718-8864-4356aae19c72 - jinja: 'Translate into Russian: {{translation["en"]}} + id: 5e34c936-eb95-49fd-abf8-76e069250e6a + jinja: 'What is the Russian translation of the English sentence: {{translation["en"]}} ||| {{translation["ru"]}}' metadata: !TemplateMetadata @@ -210,25 +197,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-target - reference: '' - 90ed694d-fa0a-45d3-b300-89675e37fa85: !Template + - gpt-3-en-ru-source+target + reference: GPT 3 paper + 62516534-2a39-47fb-8403-5c6e7dd1c97b: !Template answer_choices: null - id: 90ed694d-fa0a-45d3-b300-89675e37fa85 - jinja: 'English: {{translation["en"]}} = Russian: + id: 62516534-2a39-47fb-8403-5c6e7dd1c97b + jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version + should say: - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-ru-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - 9913cc00-d35e-42ec-807e-b32df461d0be: !Template + - version-ru-en-source+target + reference: '' + 6459ffc2-6c60-459e-82db-84199a66362d: !Template answer_choices: null - id: 9913cc00-d35e-42ec-807e-b32df461d0be + id: 6459ffc2-6c60-459e-82db-84199a66362d jinja: 'Russian: {{translation["ru"]}} = English: ||| {{translation["en"]}}' @@ -240,51 +228,37 @@ templates: name: !!python/tuple - xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 - a719f99f-bcc5-472c-923b-47138d0be28e: !Template + 7aa83e54-ffb0-4c9e-8aea-4d91237897db: !Template answer_choices: null - id: a719f99f-bcc5-472c-923b-47138d0be28e - jinja: 'What is the English translation of : {{translation["ru"]}} + id: 7aa83e54-ffb0-4c9e-8aea-4d91237897db + jinja: '{{translation["ru"]}} = English: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-ru-en-target - reference: GPT-3 paper - ae34ab89-2e9c-4f95-ac8c-0cdd5361329c: !Template + - xglm-ru-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 81b4a43f-b096-4b0e-944d-9a23f0110a2b: !Template answer_choices: null - id: ae34ab89-2e9c-4f95-ac8c-0cdd5361329c - jinja: 'Translate English into Russian: {{translation["en"]}} + id: 81b4a43f-b096-4b0e-944d-9a23f0110a2b + jinja: 'How do you say {{translation["ru"]}} in English? - ||| {{translation["ru"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-ru-source+starget + - how_to_say-ru-en-target reference: '' - b545a1ed-be0b-433e-9c51-ef902d345a4d: !Template - answer_choices: null - id: b545a1ed-be0b-433e-9c51-ef902d345a4d - jinja: '{{translation["en"]}} = Russian: - - ||| {{translation["ru"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - xglm-en-ru-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - c5f14676-23d8-4ef1-aa37-271db7a27153: !Template + 89e8ef4e-f0e9-4e61-970a-ab6e6e515387: !Template answer_choices: null - id: c5f14676-23d8-4ef1-aa37-271db7a27153 + id: 89e8ef4e-f0e9-4e61-970a-ab6e6e515387 jinja: 'Given the following passage: {{translation["ru"]}} , a good English translation is: @@ -297,52 +271,51 @@ templates: name: !!python/tuple - a_good_translation-ru-en-target reference: '' - dba4eace-b907-437a-abf7-c0d173a24711: !Template + 9488c5d3-20bb-4044-a1de-ca0bda231ad2: !Template answer_choices: null - id: dba4eace-b907-437a-abf7-c0d173a24711 - jinja: 'Translate Russian into English: {{translation["ru"]}} + id: 9488c5d3-20bb-4044-a1de-ca0bda231ad2 + jinja: 'If the original version says: {{translation["en"]}}; then the Russian + version should say: - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-ru-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - e036a9fa-5836-4c97-b52d-677d07092097: !Template + - version-en-ru-target + reference: '' + aa3c33be-469b-4a65-9aa9-3d67552b8455: !Template answer_choices: null - id: e036a9fa-5836-4c97-b52d-677d07092097 - jinja: 'If the Russian version says: {{translation["ru"]}}; then the English version - should say: - - ||| {{translation["en"]}}' + id: aa3c33be-469b-4a65-9aa9-3d67552b8455 + jinja: 'Given the following passage: {{translation["en"]}} , a good Russian translation + is: ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-ru-en-source+target + - a_good_translation-en-ru-target reference: '' - ed2b0b2f-f79c-4f15-b8ff-0b5d42e4292b: !Template + ac0c8454-6ec4-4743-b31d-768279cbf9e3: !Template answer_choices: null - id: ed2b0b2f-f79c-4f15-b8ff-0b5d42e4292b - jinja: '{{translation["ru"]}} translates into English as: + id: ac0c8454-6ec4-4743-b31d-768279cbf9e3 + jinja: 'Translate this from English into Russian: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["ru"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_ru-en-target + - translate-en-ru-source+starget reference: '' - ee346d61-67a2-4cc4-adff-6cd71eab5a16: !Template + af6bced2-3292-4e85-8434-44bf0cebc14b: !Template answer_choices: null - id: ee346d61-67a2-4cc4-adff-6cd71eab5a16 + id: af6bced2-3292-4e85-8434-44bf0cebc14b jinja: 'If the original version says: {{translation["ru"]}}; then the English version should say: @@ -355,10 +328,23 @@ templates: name: !!python/tuple - version-ru-en-target reference: '' - fae13413-02db-4c10-9b5b-43ccd35172a7: !Template + afa124a1-d7b0-4b5b-8d5d-8d20f47aa6aa: !Template answer_choices: null - id: fae13413-02db-4c10-9b5b-43ccd35172a7 - jinja: 'How do you say {{translation["ru"]}} in English? + id: afa124a1-d7b0-4b5b-8d5d-8d20f47aa6aa + jinja: 'Given the following source text in Russian: {{translation["ru"]}} , a + good English translation is: ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - a_good_translation-ru-en-source+target + reference: '' + bf638811-ea72-40e7-a9d2-68fe34e41422: !Template + answer_choices: null + id: bf638811-ea72-40e7-a9d2-68fe34e41422 + jinja: 'Russian: {{translation["ru"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -367,5 +353,19 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-ru-en-target + - translate_as_ru-en-source+target reference: '' + cc4e99c5-9172-486c-960c-5708e960210b: !Template + answer_choices: null + id: cc4e99c5-9172-486c-960c-5708e960210b + jinja: 'English: {{translation["en"]}} = Russian: + + ||| {{translation["ru"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-en-ru-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt19/zh-en/templates.yaml b/promptsource/templates/wmt19/zh-en/templates.yaml index 682684dc1..002097f7e 100644 --- a/promptsource/templates/wmt19/zh-en/templates.yaml +++ b/promptsource/templates/wmt19/zh-en/templates.yaml @@ -1,10 +1,10 @@ dataset: wmt19 subset: zh-en templates: - 036ae20f-5a46-410e-8aae-2345539dea64: !Template + 00f4d378-fd31-40e9-bddf-80c716311a55: !Template answer_choices: null - id: 036ae20f-5a46-410e-8aae-2345539dea64 - jinja: 'Translate English into Chinese: {{translation["en"]}} + id: 00f4d378-fd31-40e9-bddf-80c716311a55 + jinja: 'Translate this from English into Chinese: {{translation["en"]}} ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -15,53 +15,52 @@ templates: name: !!python/tuple - translate-en-zh-source+starget reference: '' - 1a2cbf2b-0369-4176-b9ea-aa49995db87e: !Template + 03bf5bf1-50dc-4278-ae77-c21fc9bb5b38: !Template answer_choices: null - id: 1a2cbf2b-0369-4176-b9ea-aa49995db87e - jinja: 'Translate Chinese into English: {{translation["zh"]}} + id: 03bf5bf1-50dc-4278-ae77-c21fc9bb5b38 + jinja: 'What is the Chinese translation of: {{translation["en"]}} - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-source+target - reference: https://arxiv.org/pdf/1910.10683.pdf - 24eb3c84-a760-4493-b1b3-5501eba6a1c9: !Template + - gpt-3-en-zh-target + reference: GPT-3 paper + 08d528bb-62b8-407f-a917-cd8f2f039dc0: !Template answer_choices: null - id: 24eb3c84-a760-4493-b1b3-5501eba6a1c9 - jinja: 'If the original version says: {{translation["zh"]}}; then the English - version should say: + id: 08d528bb-62b8-407f-a917-cd8f2f039dc0 + jinja: 'What is the English translation of : {{translation["zh"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-target - reference: '' - 25871786-1c7a-4cb3-a1a3-e6b0e7137e02: !Template + - gpt-3-zh-en-target + reference: GPT-3 paper + 0c961ed2-1fd4-425a-87cf-4f597464afd3: !Template answer_choices: null - id: 25871786-1c7a-4cb3-a1a3-e6b0e7137e02 - jinja: 'English: {{translation["en"]}} translates into Chinese as: + id: 0c961ed2-1fd4-425a-87cf-4f597464afd3 + jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-source+target - reference: '' - 28959a82-6a8c-4e2f-be8e-143e3f9c36fc: !Template + - gpt-3-en-zh-source+target + reference: GPT 3 paper + 13411150-33f4-48db-b6c3-e7704956d6c8: !Template answer_choices: null - id: 28959a82-6a8c-4e2f-be8e-143e3f9c36fc - jinja: 'How do you say {{translation["en"]}} in Chinese? + id: 13411150-33f4-48db-b6c3-e7704956d6c8 + jinja: 'English: {{translation["en"]}} translates into Chinese as: ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -70,27 +69,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-en-zh-target - reference: '' - 2f41e2db-f1d3-4ee0-bce2-ecd3a62b2ad7: !Template - answer_choices: null - id: 2f41e2db-f1d3-4ee0-bce2-ecd3a62b2ad7 - jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version - should say: - - ||| {{translation["zh"]}}' - metadata: !TemplateMetadata - choices_in_prompt: false - metrics: - - BLEU - original_task: true - name: !!python/tuple - - version-en-zh-source+target + - translate_as_en-zh-source+target reference: '' - 2f80c39c-5bba-4295-9175-9f0f72622045: !Template + 1e87b260-0ac0-45dd-aff6-dc1ee1403f6b: !Template answer_choices: null - id: 2f80c39c-5bba-4295-9175-9f0f72622045 - jinja: 'What is the Chinese translation of the English sentence: {{translation["en"]}} + id: 1e87b260-0ac0-45dd-aff6-dc1ee1403f6b + jinja: 'Translate this into Chinese: {{translation["en"]}} ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -99,12 +83,12 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-source+target - reference: GPT 3 paper - 30aed8d9-b822-42f9-a1b1-b18537b9df60: !Template + - translate-en-zh-target + reference: '' + 2a63c023-4fb7-41d2-bb84-5fad3bf7f6ac: !Template answer_choices: null - id: 30aed8d9-b822-42f9-a1b1-b18537b9df60 - jinja: '{{translation["zh"]}} = English: + id: 2a63c023-4fb7-41d2-bb84-5fad3bf7f6ac + jinja: 'How do you say {{translation["zh"]}} in English? ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -113,26 +97,27 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 32758794-35ba-44c4-b7cc-3dd1076b33e9: !Template + - how_to_say-zh-en-target + reference: '' + 3204009f-5f11-449b-b31c-76cf4b099025: !Template answer_choices: null - id: 32758794-35ba-44c4-b7cc-3dd1076b33e9 - jinja: 'Translate into Chinese: {{translation["en"]}} + id: 3204009f-5f11-449b-b31c-76cf4b099025 + jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version + should say: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-en-zh-target + - version-zh-en-source+target reference: '' - 327b953c-13c0-4ee5-b33b-15762767d2b8: !Template + 328efda5-1fc0-4a75-96c2-351a8669ca6b: !Template answer_choices: null - id: 327b953c-13c0-4ee5-b33b-15762767d2b8 - jinja: 'How do you say {{translation["zh"]}} in English? + id: 328efda5-1fc0-4a75-96c2-351a8669ca6b + jinja: '{{translation["zh"]}} translates into English as: ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -141,24 +126,26 @@ templates: - BLEU original_task: true name: !!python/tuple - - how_to_say-zh-en-target + - translate_as_zh-en-target reference: '' - 44148f50-2f94-4505-8fd2-8b8524f61f69: !Template + 4487fae5-dddd-482a-bdd7-91a37a7537dd: !Template answer_choices: null - id: 44148f50-2f94-4505-8fd2-8b8524f61f69 - jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation - is: ||| {{translation["zh"]}}' + id: 4487fae5-dddd-482a-bdd7-91a37a7537dd + jinja: 'If the English version says: {{translation["en"]}}; then the Chinese version + should say: + + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-en-zh-target + - version-en-zh-source+target reference: '' - 4a4e6ef6-79fb-4b59-a471-6caad2450434: !Template + 5269bb03-015a-4db0-a5fa-9337d95c81e5: !Template answer_choices: null - id: 4a4e6ef6-79fb-4b59-a471-6caad2450434 + id: 5269bb03-015a-4db0-a5fa-9337d95c81e5 jinja: 'Given the following source text in English: {{translation["en"]}} , a good Chinese translation is: @@ -171,109 +158,109 @@ templates: name: !!python/tuple - a_good_translation-en-zh-source+target reference: '' - 6c773fcf-3fd9-4a07-b0cc-2a209421666d: !Template + 6b066007-3a6b-415f-b8ef-bb93f894f831: !Template answer_choices: null - id: 6c773fcf-3fd9-4a07-b0cc-2a209421666d - jinja: 'What is the Chinese translation of: {{translation["en"]}} + id: 6b066007-3a6b-415f-b8ef-bb93f894f831 + jinja: 'English: {{translation["en"]}} = Chinese: - ||| {{translation["zh"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-en-zh-target - reference: GPT-3 paper - 744128d8-5265-4dfa-b492-38bee231db22: !Template + - xglm-en-zh-source-target + reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 + 72c78300-19a2-4aad-83e9-e5b30a46dafd: !Template answer_choices: null - id: 744128d8-5265-4dfa-b492-38bee231db22 - jinja: 'What is the English translation of : {{translation["zh"]}} + id: 72c78300-19a2-4aad-83e9-e5b30a46dafd + jinja: '{{translation["en"]}} = Chinese: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-target - reference: GPT-3 paper - 75b55155-1d0b-4d7d-918f-7cff6297f343: !Template + - xglm-en-zh-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + 743eb883-26f3-4868-b8c0-a7d5b83776c6: !Template answer_choices: null - id: 75b55155-1d0b-4d7d-918f-7cff6297f343 - jinja: 'Chinese: {{translation["zh"]}} translates into English as: + id: 743eb883-26f3-4868-b8c0-a7d5b83776c6 + jinja: 'If the original version says: {{translation["zh"]}}; then the English + version should say: - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-source+target + - version-zh-en-target reference: '' - 81879641-6d30-4a85-8cc8-ad5249564c13: !Template + 7f599e74-5e22-452d-a661-111f3388e136: !Template answer_choices: null - id: 81879641-6d30-4a85-8cc8-ad5249564c13 - jinja: '{{translation["en"]}} = Chinese: + id: 7f599e74-5e22-452d-a661-111f3388e136 + jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - 83f53c84-94e5-4a09-a9e1-910b94ac3670: !Template + - gpt-3-zh-en-source+target + reference: GPT-3 paper + 8040aec8-e681-47f3-b204-c18ec4f78f5e: !Template answer_choices: null - id: 83f53c84-94e5-4a09-a9e1-910b94ac3670 - jinja: ' {{translation["en"]}} translates into Chinese as: - - ||| {{translation["zh"]}}' + id: 8040aec8-e681-47f3-b204-c18ec4f78f5e + jinja: 'Given the following passage: {{translation["en"]}} , a good Chinese translation + is: ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_en-zh-target + - a_good_translation-en-zh-target reference: '' - 85af687a-da39-48c9-9783-8b7304bae3ea: !Template + 868c0495-d871-418d-8ff3-a8d34309721f: !Template answer_choices: null - id: 85af687a-da39-48c9-9783-8b7304bae3ea - jinja: 'If the Chinese version says: {{translation["zh"]}}; then the English version - should say: + id: 868c0495-d871-418d-8ff3-a8d34309721f + jinja: 'Translate this from Chinese into English: {{translation["zh"]}} - ||| {{translation["en"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-zh-en-source+target - reference: '' - 952df965-758a-4f8b-ae08-2ccbd12b66c0: !Template + - translate-zh-en-source+target + reference: https://arxiv.org/pdf/1910.10683.pdf + 88bd8591-5b26-4f2b-9a12-8e57995bbaab: !Template answer_choices: null - id: 952df965-758a-4f8b-ae08-2ccbd12b66c0 - jinja: 'Translate into English: {{translation["zh"]}} + id: 88bd8591-5b26-4f2b-9a12-8e57995bbaab + jinja: 'If the original version says: {{translation["en"]}}; then the Chinese + version should say: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate-zh-en-target + - version-en-zh-target reference: '' - 9c27aeb5-b36f-4df9-81b3-c231d7f2a158: !Template + 89ee6d56-84c2-4e44-8b70-2da7a8054f9b: !Template answer_choices: null - id: 9c27aeb5-b36f-4df9-81b3-c231d7f2a158 - jinja: 'English: {{translation["en"]}} = Chinese: + id: 89ee6d56-84c2-4e44-8b70-2da7a8054f9b + jinja: 'How do you say {{translation["en"]}} in Chinese? ||| {{translation["zh"]}}' metadata: !TemplateMetadata @@ -282,38 +269,38 @@ templates: - BLEU original_task: true name: !!python/tuple - - xglm-en-zh-source-target - reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 - a85d88b4-d190-4d86-8f35-685b54c32af0: !Template + - how_to_say-en-zh-target + reference: '' + 98dde747-820b-4ec9-b9ab-fc76aba0c867: !Template answer_choices: null - id: a85d88b4-d190-4d86-8f35-685b54c32af0 - jinja: '{{translation["zh"]}} translates into English as: - - ||| {{translation["en"]}}' + id: 98dde747-820b-4ec9-b9ab-fc76aba0c867 + jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a + good English translation is: ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - translate_as_zh-en-target + - a_good_translation-zh-en-source+target reference: '' - b11cc24d-867b-4e0a-a19b-e7e1b518efaf: !Template + ac3760e1-d074-43d4-8199-b19a1ee3effb: !Template answer_choices: null - id: b11cc24d-867b-4e0a-a19b-e7e1b518efaf - jinja: 'Given the following source text in Chinese: {{translation["zh"]}} , a - good English translation is: ||| {{translation["en"]}}' + id: ac3760e1-d074-43d4-8199-b19a1ee3effb + jinja: 'Chinese: {{translation["zh"]}} translates into English as: + + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - a_good_translation-zh-en-source+target + - translate_as_zh-en-source+target reference: '' - c12af803-17f0-4add-8cbe-49ebbadcbb90: !Template + bccb49ca-d1b7-4930-b02d-52d8cbfbe6a5: !Template answer_choices: null - id: c12af803-17f0-4add-8cbe-49ebbadcbb90 + id: bccb49ca-d1b7-4930-b02d-52d8cbfbe6a5 jinja: 'Given the following passage: {{translation["zh"]}} , a good English translation is: @@ -326,24 +313,24 @@ templates: name: !!python/tuple - a_good_translation-zh-en-target reference: '' - eacc82a6-ad5d-42c0-a1a4-ff276751d713: !Template + bd22ec10-1e72-45d0-93b0-edc16a567097: !Template answer_choices: null - id: eacc82a6-ad5d-42c0-a1a4-ff276751d713 - jinja: 'Chinese: {{translation["zh"]}} = English: + id: bd22ec10-1e72-45d0-93b0-edc16a567097 + jinja: ' {{translation["en"]}} translates into Chinese as: - ||| {{translation["en"]}}' + ||| {{translation["zh"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - xglm-zh-en-source+target - reference: XGLM paper https://arxiv.org/abs/2112.10668 - f5315dac-8ab1-44e5-9fc7-3d84db8f69cf: !Template + - translate_as_en-zh-target + reference: '' + e1275325-ec0c-407f-a13d-903260f66347: !Template answer_choices: null - id: f5315dac-8ab1-44e5-9fc7-3d84db8f69cf - jinja: 'What is the English translation of the Chinese sentence: {{translation["zh"]}} + id: e1275325-ec0c-407f-a13d-903260f66347 + jinja: 'Translate this into English: {{translation["zh"]}} ||| {{translation["en"]}}' metadata: !TemplateMetadata @@ -352,20 +339,33 @@ templates: - BLEU original_task: true name: !!python/tuple - - gpt-3-zh-en-source+target - reference: GPT-3 paper - f77ba2f6-01e8-46f2-aac7-b99af59dfb94: !Template + - translate-zh-en-target + reference: '' + e196c412-aeea-4a1d-bdcb-992734772e07: !Template answer_choices: null - id: f77ba2f6-01e8-46f2-aac7-b99af59dfb94 - jinja: 'If the original version says: {{translation["en"]}}; then the Chinese - version should say: + id: e196c412-aeea-4a1d-bdcb-992734772e07 + jinja: '{{translation["zh"]}} = English: - ||| {{translation["zh"]}}' + ||| {{translation["en"]}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - BLEU original_task: true name: !!python/tuple - - version-en-zh-target - reference: '' + - xglm-zh-en-target + reference: XGLM paper https://arxiv.org/abs/2112.10668 + e4aaa404-3e92-4469-a074-b671442e049f: !Template + answer_choices: null + id: e4aaa404-3e92-4469-a074-b671442e049f + jinja: 'Chinese: {{translation["zh"]}} = English: + + ||| {{translation["en"]}}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + original_task: true + name: !!python/tuple + - xglm-zh-en-source+target + reference: XGLM paper https://arxiv.org/abs/2112.10668 From 2608a3828e3512514fc60e50fd4146fac7da5a11 Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Wed, 29 Jun 2022 18:47:41 +0000 Subject: [PATCH 15/15] fix names --- .../templates/wmt14/cs-en/templates.yaml | 78 +++++++------------ .../templates/wmt14/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt14/hi-en/templates.yaml | 78 +++++++------------ .../templates/wmt14/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt15/cs-en/templates.yaml | 78 +++++++------------ .../templates/wmt15/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt15/fi-en/templates.yaml | 78 +++++++------------ .../templates/wmt15/fr-en/templates.yaml | 78 +++++++------------ .../templates/wmt15/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/cs-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/fi-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/ro-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt16/tr-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/cs-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/fi-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/lv-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/tr-en/templates.yaml | 78 +++++++------------ .../templates/wmt17/zh-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/cs-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/et-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/fi-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/tr-en/templates.yaml | 78 +++++++------------ .../templates/wmt18/zh-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/de-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/fi-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/fr-de/templates.yaml | 78 +++++++------------ .../templates/wmt19/gu-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/kk-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/lt-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/ru-en/templates.yaml | 78 +++++++------------ .../templates/wmt19/zh-en/templates.yaml | 78 +++++++------------ 37 files changed, 962 insertions(+), 1924 deletions(-) diff --git a/promptsource/templates/wmt14/cs-en/templates.yaml b/promptsource/templates/wmt14/cs-en/templates.yaml index 8f429568a..ad66226e6 100644 --- a/promptsource/templates/wmt14/cs-en/templates.yaml +++ b/promptsource/templates/wmt14/cs-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-source+target + name: translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 16da9664-30e5-4acc-aff6-21658158cf85: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-target + name: gpt-3-en-cs-target reference: GPT-3 paper 3b3ca652-8414-4442-b55f-c2fc7630b071: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-source+target + name: a_good_translation-en-cs-source+target reference: '' 4266e99a-41fa-4c2d-8bbc-91317f42d9ff: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-source+target + name: gpt-3-cs-en-source+target reference: GPT-3 paper 46621847-e018-43fe-b8d1-af439e130254: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-source+target + name: gpt-3-en-cs-source+target reference: GPT 3 paper 4c52dc6a-f16c-4f3c-99d0-274b4162a616: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-target + name: version-en-cs-target reference: '' 5b87693b-ed41-4794-bb3e-b9bf7ada2689: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-target + name: a_good_translation-en-cs-target reference: '' 5e476fb7-5e6e-4732-a084-88e361a82de0: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-target + name: translate_as_cs-en-target reference: '' 744bab4b-44d1-4368-a98d-4a403c1a7e0e: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-target + name: translate-cs-en-target reference: '' 7fd9d8d7-0c72-42da-a029-ef4d9a9e5f72: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-source+target + name: a_good_translation-cs-en-source+target reference: '' 809cc7a3-189b-4018-8f81-1e65c2f42271: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-target + name: a_good_translation-cs-en-target reference: '' 9023b083-ccf1-42a4-9589-5fffd3562649: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-source+target + name: version-cs-en-source+target reference: '' 92e3a4a7-e7c2-4ea1-abfb-be63069d13fa: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-target + name: translate-en-cs-target reference: '' a63daae0-77ba-4790-a89d-2428193fbcdc: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-target + name: gpt-3-cs-en-target reference: GPT-3 paper a63fd691-4f07-41da-a98c-a934e866e74e: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-target + name: xglm-cs-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 ab264706-636e-4bd0-a762-860029b54f45: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-source+starget + name: translate-en-cs-source+starget reference: '' ac25c74b-1a81-409e-9c4f-bdc94912721c: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-target + name: translate_as_en-cs-target reference: '' b42858a8-35d7-4172-ace4-80df1f6a94e0: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-source+target + name: version-en-cs-source+target reference: '' c4629789-74ce-4378-8d03-f75b2327ad77: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-source-target + name: xglm-en-cs-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 cdce6dd9-52ad-42c3-a691-8efb13ab535f: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-source+target + name: translate_as_cs-en-source+target reference: '' dcefc4dc-560d-4bf5-8623-c99d535232b0: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-cs-en-target + name: how_to_say-cs-en-target reference: '' e13ee973-fce4-4d42-8ec5-b871c20eaf39: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-source+target + name: xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 ee298b4c-8165-44cc-81b6-1fb18f4e9e02: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-source+target + name: translate_as_en-cs-source+target reference: '' f7af3145-3444-4183-921b-a549fef5e1b3: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-target + name: version-cs-en-target reference: '' f7d630ca-1791-4aaf-97b5-7c15e5eba296: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-cs-target + name: how_to_say-en-cs-target reference: '' f83cb5c3-7989-49d9-a1cb-6fdc16b445e3: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-target + name: xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt14/de-en/templates.yaml b/promptsource/templates/wmt14/de-en/templates.yaml index 389dc09c0..655638bd9 100644 --- a/promptsource/templates/wmt14/de-en/templates.yaml +++ b/promptsource/templates/wmt14/de-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' 1bd45cc7-cfe0-4ad9-a157-e585b7931afe: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 1d5e9684-1964-45f9-90fd-5e6bde7d22f4: !Template answer_choices: null @@ -42,8 +40,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' 1dcd49d2-0b77-4a6e-907b-c39453282bfe: !Template answer_choices: null @@ -56,8 +53,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 27a87c3b-1516-4a1c-b834-141b11b85f20: !Template answer_choices: null @@ -70,8 +66,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' 2f928e28-04d8-4538-bea4-545a072173ed: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' 3210d998-8cd4-420c-9ea4-3fa3c67aa5f7: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper 36d02443-77b4-4f6b-9220-838f996b2f28: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 42dc9689-b971-4641-b6e3-97f55e6da9b7: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 44ecf43e-f4ae-4718-a730-945e86777917: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' 54d4f276-b5f4-4895-8e62-fc23362670b0: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' 6e69d25e-6209-41d3-b2ee-6d18c861f34f: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' 719762e7-0928-40fc-8c75-274c99586135: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' 886c41e7-cbc7-405f-8d8c-f6d3f11ee473: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 8bc4714f-f068-4d48-a661-0666c2f66193: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' 8ec97fe5-268c-4a3f-a173-1bbaa0d2104f: !Template answer_choices: null @@ -226,8 +211,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper 91d21ef8-c0c9-473d-b4e0-ba962664cd90: !Template answer_choices: null @@ -241,8 +225,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' 926fb8b4-d2a5-428c-aee0-fac54586bcc7: !Template answer_choices: null @@ -255,8 +238,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' a7f186b5-3ec2-46b6-824a-46a68f102514: !Template answer_choices: null @@ -269,8 +251,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 ae772d49-5a93-4ffd-b1a7-9e2034a23509: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' aeb84e76-e929-4dd6-8002-69511c2e9fb6: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' b57234a1-0d63-40b3-909c-14c8a8896c7c: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' c6d34117-0d80-4e19-a669-481293148bab: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper d04ece63-a018-4f93-8b2b-e8d48ed239e8: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' dc6bcb43-5b88-4b16-9863-51bbad17ddd6: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper eaee5677-f889-4bfd-8fbb-ad976b917506: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' diff --git a/promptsource/templates/wmt14/hi-en/templates.yaml b/promptsource/templates/wmt14/hi-en/templates.yaml index cf9bb90d8..a6c777324 100644 --- a/promptsource/templates/wmt14/hi-en/templates.yaml +++ b/promptsource/templates/wmt14/hi-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-hi-en-target + name: version-hi-en-target reference: '' 03664fac-67ef-414d-8e4a-504ad4d7a8a0: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-hi-en-source+target + name: a_good_translation-hi-en-source+target reference: '' 1c4fc122-d6e1-41f5-91e8-1aacbbfb3549: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-hi-en-source+target + name: gpt-3-hi-en-source+target reference: GPT-3 paper 1dcf0a6e-1463-4c07-8431-95ba24273257: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-hi-target + name: how_to_say-en-hi-target reference: '' 25daf37e-e684-419e-a250-bdeeb82a7df6: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-hi-target + name: xglm-en-hi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 267ca01b-599f-46d6-a85d-ce1915382c18: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-hi-en-source+target + name: translate-hi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 2fd2a8ba-ea2a-4686-bee5-2f49fb31ebca: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-hi-source+starget + name: translate-en-hi-source+starget reference: '' 40d4d47a-fd09-48af-88d2-ef3ee7a377d6: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-hi-en-target + name: translate-hi-en-target reference: '' 42379c42-04c5-4ea9-99ca-f43f1b1cfc1b: !Template answer_choices: null @@ -123,8 +115,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-hi-target + name: a_good_translation-en-hi-target reference: '' 42dd0622-3e2a-4fa0-b3fc-6087225503c9: !Template answer_choices: null @@ -137,8 +128,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-hi-en-source+target + name: xglm-hi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 5149f4af-3272-42dd-a4d5-d3cd83927655: !Template answer_choices: null @@ -151,8 +141,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-hi-en-target + name: how_to_say-hi-en-target reference: '' 54a82bf0-4765-4e10-9311-1f07f6c22799: !Template answer_choices: null @@ -165,8 +154,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-hi-source-target + name: xglm-en-hi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 678d458a-f40c-480c-811f-f8aa51cda71a: !Template answer_choices: null @@ -179,8 +167,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-hi-target + name: translate-en-hi-target reference: '' 8116f3dd-3066-4a47-9a0c-92884d98f6c2: !Template answer_choices: null @@ -193,8 +180,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_hi-en-target + name: translate_as_hi-en-target reference: '' 84b903ab-e138-4975-8507-77be58bc0fb0: !Template answer_choices: null @@ -207,8 +193,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-hi-source+target + name: translate_as_en-hi-source+target reference: '' 9b430f52-31a1-4b7b-9600-59069a706b2c: !Template answer_choices: null @@ -221,8 +206,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-hi-en-target + name: xglm-hi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 9f9c17e6-25b3-4240-b0d3-61f47b9c2523: !Template answer_choices: null @@ -235,8 +219,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_hi-en-source+target + name: translate_as_hi-en-source+target reference: '' a5d28474-3b52-4723-98e5-1330ca121381: !Template answer_choices: null @@ -250,8 +233,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-hi-en-source+target + name: version-hi-en-source+target reference: '' ab0195bd-4abd-4d9e-8107-afa7a3a6f6fc: !Template answer_choices: null @@ -264,8 +246,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-hi-en-target + name: gpt-3-hi-en-target reference: GPT-3 paper acc9d2cb-e156-4907-a9ae-9b85b714ce19: !Template answer_choices: null @@ -278,8 +259,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-hi-target + name: translate_as_en-hi-target reference: '' b5952cac-9388-4901-98ed-c45cccfed5de: !Template answer_choices: null @@ -293,8 +273,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-hi-target + name: version-en-hi-target reference: '' d1e354a7-8fa3-415a-9bb7-755e1ae21813: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-hi-source+target + name: a_good_translation-en-hi-source+target reference: '' d3796afb-95f9-4a69-b053-1340e3b86626: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-hi-source+target + name: gpt-3-en-hi-source+target reference: GPT 3 paper eb27a29c-e238-4ebd-a675-456d2102a80e: !Template answer_choices: null @@ -336,8 +313,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-hi-target + name: gpt-3-en-hi-target reference: GPT-3 paper f5e8dd73-6e5c-48a7-b92a-766b5ce9b5e5: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-hi-source+target + name: version-en-hi-source+target reference: '' fbd2d598-80e9-4ce6-b85e-fb269aa82580: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-hi-en-target + name: a_good_translation-hi-en-target reference: '' diff --git a/promptsource/templates/wmt14/ru-en/templates.yaml b/promptsource/templates/wmt14/ru-en/templates.yaml index b35f56109..bd047d7d7 100644 --- a/promptsource/templates/wmt14/ru-en/templates.yaml +++ b/promptsource/templates/wmt14/ru-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper 073879bd-c1ca-429e-89e6-8e8ad608b91b: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' 255dcbb5-6d94-4a71-b868-1ada8b050c84: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' 2fa8c0fd-cdf5-471d-96ca-c24768ac0409: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' 333bc458-f648-47c5-8bd9-699c901da9c8: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 38d575d3-b90c-4b53-b65c-91fd94196282: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 3afd655a-42c2-4a47-95ef-193ded37e470: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 4dad78cb-8811-4319-be5b-d8dc9380c0ac: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 7b30c457-c1eb-4ac2-8c61-9edfff3aa89e: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' 7d060eb3-ae66-4479-89b3-666af83e3cbb: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' 85f57cfb-454e-44b8-9065-a19707060565: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' 8aed1935-84a3-423e-945c-3988d59dae43: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' 9306ac01-a84c-4219-aa7c-5b33470597b5: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' 98364940-cbc9-4676-ae98-83133f607c96: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 9cf0e07b-bd5a-4c86-968c-5e36ec75cde1: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' b29ed609-3647-4687-88c9-8356c19b0703: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 bfe5a2cd-c52c-4703-929e-db381f2e2932: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' c78d918c-c3dc-4c53-a71c-21e54480f8a7: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper d4b1889e-398d-4b01-a597-4f5e11508d40: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' d5ed24af-fd55-4788-b352-114684bad41f: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper d9368e60-ae18-4c89-b1fb-823cdc2bbae3: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' ed05f70a-7dea-4eec-946f-061a2485e5a9: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' f2c473df-a28e-493f-82f3-d49acf39f49b: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 f71190c6-4297-4e4a-8c51-165d711c51c2: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' f901c23f-34f6-4512-8682-e06db86e2f76: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' f959abd8-ee89-419f-9c8e-293ec9966f68: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' diff --git a/promptsource/templates/wmt15/cs-en/templates.yaml b/promptsource/templates/wmt15/cs-en/templates.yaml index 55b07af7e..0b5852be8 100644 --- a/promptsource/templates/wmt15/cs-en/templates.yaml +++ b/promptsource/templates/wmt15/cs-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-source-target + name: xglm-en-cs-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 141683d7-7150-46e3-98e5-4e626b7a8eac: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-target + name: gpt-3-cs-en-target reference: GPT-3 paper 157bda68-3bf8-4e13-8a7a-53ad56875938: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-target + name: xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 187ea6b9-d94c-4982-b135-9dfa865ed640: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-target + name: version-en-cs-target reference: '' 422e07ed-5982-43ff-958f-8523b70a21cb: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-source+target + name: translate_as_cs-en-source+target reference: '' 549a16e8-f5d0-4364-80f1-e7406ee4ae3a: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-target + name: translate_as_cs-en-target reference: '' 595eefb8-b237-452f-b165-f082a280faae: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-source+target + name: xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 607dceeb-1034-43fc-8101-0aec68caeb7d: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-source+target + name: a_good_translation-cs-en-source+target reference: '' 643bf184-cdd4-4cf5-99d6-85cbc1f20a61: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-source+starget + name: translate-en-cs-source+starget reference: '' 66ccaa8f-7ee4-4fd3-ac02-9caf6ccd821b: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-cs-target + name: how_to_say-en-cs-target reference: '' 6995c2de-c2a5-43e3-9b6e-04a2a8d53ad7: !Template answer_choices: null @@ -152,8 +142,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-cs-en-target + name: how_to_say-cs-en-target reference: '' 7287aaeb-aaab-4596-9f77-3e1bc1552be0: !Template answer_choices: null @@ -166,8 +155,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-target + name: xglm-cs-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 78823d0a-923e-409e-9fd4-cc43e6403956: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-target + name: gpt-3-en-cs-target reference: GPT-3 paper 7aed74f4-94a2-4f64-b169-8032fc298062: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-target + name: a_good_translation-cs-en-target reference: '' 7d5fe968-366f-4bc3-b744-625167a8a632: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-source+target + name: translate_as_en-cs-source+target reference: '' 876283e3-6536-4bdb-8d0d-bba32f4109bd: !Template answer_choices: null @@ -223,8 +208,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-target + name: translate-en-cs-target reference: '' 87cecfcc-00d2-4ea8-a64d-b51a1465f187: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-source+target + name: translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 8c17680c-f022-4b7e-93d0-bdd58ab0f837: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-target + name: version-cs-en-target reference: '' 8f883e31-d357-4abe-a875-dbcf372024b3: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-source+target + name: version-cs-en-source+target reference: '' a78ce014-6c0a-4575-943b-d7e1dd41a2f4: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-source+target + name: a_good_translation-en-cs-source+target reference: '' bbb4d21d-6c03-4927-abc4-54d5a79ef525: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-target + name: translate-cs-en-target reference: '' d490b18b-730c-4d3f-93fc-0b3c30734b0a: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-source+target + name: gpt-3-cs-en-source+target reference: GPT-3 paper d70ec9f3-5012-479a-ba66-d8a93f545c9c: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-target + name: a_good_translation-en-cs-target reference: '' da5f5196-e8f6-4e11-a88d-c0647bd011b1: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-source+target + name: gpt-3-en-cs-source+target reference: GPT 3 paper dd852efc-969c-484c-a89e-a8998c5b8780: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-source+target + name: version-en-cs-source+target reference: '' f25d3c08-b0d7-4a65-82d3-c6bcf498c037: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-target + name: translate_as_en-cs-target reference: '' diff --git a/promptsource/templates/wmt15/de-en/templates.yaml b/promptsource/templates/wmt15/de-en/templates.yaml index cfd49d8ee..c88d62add 100644 --- a/promptsource/templates/wmt15/de-en/templates.yaml +++ b/promptsource/templates/wmt15/de-en/templates.yaml @@ -11,8 +11,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' 1066a885-9717-4a68-aef7-6b0f0ad1c74e: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 20816ff7-ba5e-4129-9a91-fc6506f0bed3: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' 246427e8-6adf-44d2-94a7-659d5baf4db6: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' 2528521d-91df-49c8-851f-e015b0955c91: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 2af25b0e-1d10-4204-a82c-27d56def4ef5: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' 37c9b82e-93be-4870-b416-f25c0c032e84: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' 3bb8e23f-cdd1-4362-91ea-ab99be1e4bd8: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' 435c6647-3672-4d22-b30b-2a71726c5d8e: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 4ea23ed8-a255-4566-8aed-9503cbc87453: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' 4f1dbc62-fa8c-4dc8-9a2e-1cad5002a38a: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 50a8f74f-bcf8-4a37-9cef-6c1be1f5a7b2: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper 5149a3ac-50b5-4a48-b3a4-8a28d1045d08: !Template answer_choices: null @@ -181,8 +169,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 5490f317-04cf-413f-b3e8-17e56521bae6: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 5d39d71b-1297-4dac-aecc-118ea4099acb: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' 6441f650-a268-4249-8e47-07c2c2e599b0: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' 7963c1bc-b551-4177-adba-ebf69fc11f63: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' 9730bf8b-7079-4012-9a0e-9449bcfc4145: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' a4565b2f-b5ad-4050-8268-91222d710589: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' b306f042-211d-4c11-8614-0a78a5e2fe04: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' dc98ad6a-8ee1-4d22-80ae-e5147d69f40e: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' e3801439-211f-47cc-8560-f7a5e4af96ae: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' e4f5fa03-4f06-4267-ba5c-24552360f7d5: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper e5073dfc-a334-4e03-833a-4ef16962c115: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper e65e6ca8-843d-4095-87a2-6f2609595568: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper fac9c78f-9cae-4a4e-a0ab-8eecd2112dad: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' diff --git a/promptsource/templates/wmt15/fi-en/templates.yaml b/promptsource/templates/wmt15/fi-en/templates.yaml index c660869ab..14a355498 100644 --- a/promptsource/templates/wmt15/fi-en/templates.yaml +++ b/promptsource/templates/wmt15/fi-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-target + name: gpt-3-en-fi-target reference: GPT-3 paper 17b3bdcd-c560-4b96-8a8d-9aeb1e5d8103: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-target + name: gpt-3-fi-en-target reference: GPT-3 paper 1df95437-89f7-4a1f-bb5c-aeb2ce7a182a: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-source+target + name: a_good_translation-en-fi-source+target reference: '' 1f6c7e70-f315-47b7-b92f-fc4ca443fed1: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fi-en-target + name: how_to_say-fi-en-target reference: '' 20cd95bc-57d5-4c81-9d00-4a97cfb55ef2: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-target + name: xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 2db28522-02e1-4bef-9aeb-5d27ba926476: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-source+target + name: a_good_translation-fi-en-source+target reference: '' 44dfc96b-6afb-44f6-9ede-df79e8c82295: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-target + name: version-en-fi-target reference: '' 55f776fa-9658-476d-8c73-8fd6b0f4b9a7: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-target + name: xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 5d63fd6d-1ab2-4da0-af39-0cfba5c3ad7b: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-target + name: translate_as_en-fi-target reference: '' 5d683d0a-a670-4bb3-90ff-8a6f059f3e2a: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-source+target + name: translate_as_fi-en-source+target reference: '' 5f065433-4860-4186-be1c-aed3d930b6fb: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-target + name: version-fi-en-target reference: '' 7519528b-39d1-45d1-911e-144615c05712: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-source+target + name: translate_as_en-fi-source+target reference: '' 7a8d7325-d4b5-46b3-9ad4-10a3290e629e: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-source+target + name: gpt-3-en-fi-source+target reference: GPT 3 paper 9013a8e5-d4f5-4ff7-a300-fd2f1deda398: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fi-target + name: how_to_say-en-fi-target reference: '' 97939122-422c-4af0-b0f7-f148fb2db9f7: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-target + name: translate-en-fi-target reference: '' 97fb6b3b-9bdc-43cb-9242-cc142e88f6a4: !Template answer_choices: null @@ -223,8 +208,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-target + name: a_good_translation-en-fi-target reference: '' 998043df-dc01-482d-a2b4-0fdd6bb3a961: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-target + name: translate_as_fi-en-target reference: '' b259f666-d816-4e61-b887-faab5638ba41: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-source+target + name: version-en-fi-source+target reference: '' b9707d6d-316a-4c05-b8a1-ebd0f1d5faa0: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-target + name: translate-fi-en-target reference: '' bfce0871-2f4d-4ccb-924f-c0d4ad45603c: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-source+target + name: translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf c28bb50c-9efd-4a4e-aff4-7d6723d2f662: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-source+target + name: gpt-3-fi-en-source+target reference: GPT-3 paper ce59f28d-c0af-40cc-bf5f-98c0ffb58bee: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-source-target + name: xglm-en-fi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 db513d6f-7cce-4231-b7b7-3b89b4f75a14: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-source+target + name: version-fi-en-source+target reference: '' e708a6dd-ebd3-4da5-be69-2bc8c09e0b53: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-source+starget + name: translate-en-fi-source+starget reference: '' efacec10-b386-4cda-b887-fd1402eefc48: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-source+target + name: xglm-fi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 f7e9b639-b6de-4043-9cc2-a8cf15180140: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-target + name: a_good_translation-fi-en-target reference: '' diff --git a/promptsource/templates/wmt15/fr-en/templates.yaml b/promptsource/templates/wmt15/fr-en/templates.yaml index 01f1abace..a4b424422 100644 --- a/promptsource/templates/wmt15/fr-en/templates.yaml +++ b/promptsource/templates/wmt15/fr-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fr-en-target + name: gpt-3-fr-en-target reference: GPT-3 paper 084baa02-db47-4d16-941a-ddcc351cd836: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fr-source+target + name: a_good_translation-en-fr-source+target reference: '' 099ffec8-8348-4a86-90a9-5095e19b87cf: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fr-en-source+target + name: translate-fr-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 0e3b43b4-16af-4f03-b9c1-ad6bfd0e243b: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fr-source+starget + name: translate-en-fr-source+starget reference: '' 13acc84f-1bcb-442d-b96a-d087eb7d45a0: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fr-target + name: how_to_say-en-fr-target reference: '' 3a6fd6dc-61a9-4c7c-acc8-57099cc3aea9: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fr-source+target + name: translate_as_en-fr-source+target reference: '' 3c7f2aa6-63b1-4ecf-b493-a81c3e56e739: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fr-en-target + name: translate_as_fr-en-target reference: '' 443e9acf-50df-43f4-9943-f8bdcca3bdef: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fr-en-source+target + name: gpt-3-fr-en-source+target reference: GPT-3 paper 4ce9b78d-ed60-45cc-8125-242a2b3c5bf4: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fr-en-target + name: translate-fr-en-target reference: '' 6b5bd444-941c-4a16-9094-9610ca459312: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fr-target + name: translate-en-fr-target reference: '' 6d7d61fd-cd2c-446e-a0ff-62b964d0355f: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fr-en-source+target + name: translate_as_fr-en-source+target reference: '' 77c63fad-d820-4d4e-95ce-1497a6176499: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fr-en-source+target + name: version-fr-en-source+target reference: '' 7955fc1f-6171-483f-accc-d840df1397df: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fr-target + name: gpt-3-en-fr-target reference: GPT-3 paper 7a84aa1f-eca4-4b85-b3f3-ea46a7c89915: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fr-source+target + name: version-en-fr-source+target reference: '' 927490ac-1935-4525-8145-17b8bfcdabe1: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fr-target + name: version-en-fr-target reference: '' c05b6121-4f46-4088-a5c5-cd1f7da75340: !Template answer_choices: null @@ -226,8 +211,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fr-source+target + name: gpt-3-en-fr-source+target reference: GPT 3 paper c26db731-296e-405e-8de7-b9b49fce8407: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fr-target + name: a_good_translation-en-fr-target reference: '' c5e6c439-c0b0-4405-ab3d-22de69b3a82b: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fr-target + name: xglm-en-fr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 cf629f0f-cba7-4623-9e98-d972b56b5b5d: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fr-en-target + name: a_good_translation-fr-en-target reference: '' d76db700-1b29-4ad6-aa04-17fece6a15b0: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fr-en-source+target + name: xglm-fr-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 dba384a3-65d1-41de-96da-325a9e65a137: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fr-source-target + name: xglm-en-fr-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 e24ed618-f906-48d3-b9d0-9dd6c2864905: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fr-en-target + name: xglm-fr-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 e971a8d9-b190-4c99-95fc-f0db73d99928: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fr-en-target + name: version-fr-en-target reference: '' f341ebe5-3f02-477b-bbc1-33a1d0750062: !Template answer_choices: null @@ -339,8 +316,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fr-target + name: translate_as_en-fr-target reference: '' f7e45e6b-e002-4cdf-a0f2-40b38f678697: !Template answer_choices: null @@ -353,8 +329,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fr-en-target + name: how_to_say-fr-en-target reference: '' f960ea4e-fd15-4c7d-baf5-139ccd5ca6b3: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fr-en-source+target + name: a_good_translation-fr-en-source+target reference: '' diff --git a/promptsource/templates/wmt15/ru-en/templates.yaml b/promptsource/templates/wmt15/ru-en/templates.yaml index 6dbedaf75..ad768b6cd 100644 --- a/promptsource/templates/wmt15/ru-en/templates.yaml +++ b/promptsource/templates/wmt15/ru-en/templates.yaml @@ -11,8 +11,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' 232268c9-bd22-4928-9ebf-db6a91cb63a8: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' 25ab3b07-66e0-4a1a-80e1-dc710f140595: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 33fb869b-2517-4761-a1d9-66de39c8362a: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' 4557e646-2161-46c9-a4a6-ad93ffcf061b: !Template answer_choices: null @@ -70,8 +66,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 46d28534-cdcf-4945-a1d8-2369f20d6eed: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 4d9c4540-cfd7-4b06-91ef-cb4391e823e6: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' 60a5e016-58ad-492c-8b23-0a1d9e3a1652: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper 71527c51-6dcd-4a0e-b31a-1e013b381911: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' 743a939d-388f-4af6-b67e-d7a527beb9f5: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' 828c119b-bee4-4d63-bb2b-6641bcc65d4b: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' 86784af8-e562-4f24-b31a-58498e7f504c: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' 88e0b171-ad44-4016-86dd-c255a9a41bbc: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 8df21afa-d98a-46a3-8976-a8609c94685b: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 975e05e6-4a83-45b3-b9ab-91eaad7e7be9: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper 993c7951-af88-4162-a007-070e297bffac: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' 9f689169-2eee-4910-b8da-0913caceb1db: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 aaf1e960-8b2f-4550-95c9-9ac618c26b1a: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' b87d7baa-985e-439c-8d19-6c22cab8e0dd: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' c00e71ea-ce5e-4559-923e-5a8d91ad3bc9: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' ce31c299-7ff7-44ac-a120-d0c0db44ff10: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' d2d02ff0-dcfe-4fdd-a774-e00e75121629: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf d6099b4c-f38a-49e1-9666-c6cd51d27115: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' dafb6762-44ba-4635-9c08-211efd746eec: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper e8a591e9-af99-4144-975a-1c05d692a6a6: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' f5796829-56c0-45cb-8036-a536b4f38409: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' diff --git a/promptsource/templates/wmt16/cs-en/templates.yaml b/promptsource/templates/wmt16/cs-en/templates.yaml index b25b2c900..b613a478e 100644 --- a/promptsource/templates/wmt16/cs-en/templates.yaml +++ b/promptsource/templates/wmt16/cs-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-target + name: xglm-cs-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 06f4d626-b17b-431f-9a28-b06307824ebe: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-target + name: translate-en-cs-target reference: '' 09d83b14-f487-4ddc-8096-5026e7e5dcd9: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-target + name: version-en-cs-target reference: '' 12eaeafb-14f3-4852-8752-949a9e5dd782: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-target + name: gpt-3-en-cs-target reference: GPT-3 paper 22053344-0169-4a3b-a021-bde387866fa6: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-source+target + name: translate_as_en-cs-source+target reference: '' 3037d57c-496d-4b2c-9f96-4965092d0a9d: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-source+target + name: version-en-cs-source+target reference: '' 4b8ac0e0-c902-4814-9dae-23f09db04f1d: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-source+target + name: translate_as_cs-en-source+target reference: '' 5600ba5b-2d7d-47ef-9f51-381dc6b8a2c6: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-source+target + name: gpt-3-en-cs-source+target reference: GPT 3 paper 5a68c1d5-1780-4ae9-9d2b-de7bd20b359b: !Template answer_choices: null @@ -127,8 +119,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-source+target + name: version-cs-en-source+target reference: '' 6fba80b6-98f0-470d-b8d3-0dcfd0a51c76: !Template answer_choices: null @@ -141,8 +132,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-source-target + name: xglm-en-cs-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 74ec8367-77bf-4514-8a5b-831797eff523: !Template answer_choices: null @@ -155,8 +145,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-target + name: translate_as_cs-en-target reference: '' 7de95a1a-a966-4126-b570-48827f077a4b: !Template answer_choices: null @@ -170,8 +159,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-source+target + name: a_good_translation-en-cs-source+target reference: '' 8c8b8182-ed7f-489d-a58f-9617dada0e8c: !Template answer_choices: null @@ -184,8 +172,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-source+starget + name: translate-en-cs-source+starget reference: '' 8dbff6c3-6ac0-42b9-a749-4283bd209449: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-target + name: a_good_translation-en-cs-target reference: '' 94a10abc-f80f-4368-b1ec-2569eadf8b59: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-source+target + name: gpt-3-cs-en-source+target reference: GPT-3 paper a1ee72ef-4af0-4492-abbd-c7792e59aa77: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-target + name: translate_as_en-cs-target reference: '' b483a635-8ee9-47dc-9607-84dd22ba575e: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-target + name: translate-cs-en-target reference: '' b653b9fc-1f3a-4eb4-a93f-42d5bd624abe: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-source+target + name: xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 c8197dd1-a836-472b-a703-6ed741cc735f: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-target + name: a_good_translation-cs-en-target reference: '' c83d3b75-f46e-4f24-8305-7f495f6c2546: !Template answer_choices: null @@ -283,8 +264,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-target + name: version-cs-en-target reference: '' ce513163-bf94-419a-bf2f-fdef00124e8f: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-cs-target + name: how_to_say-en-cs-target reference: '' ddd2c29e-ca72-4f16-9189-c014a19126e1: !Template answer_choices: null @@ -311,8 +290,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-cs-en-target + name: how_to_say-cs-en-target reference: '' e3b1359f-d3f8-4dc1-977b-bb1d72066a4d: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-source+target + name: translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf ee26f75c-9ac8-4f60-a8a4-9b517c39e27e: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-source+target + name: a_good_translation-cs-en-source+target reference: '' f05efb50-36c1-4e9c-a074-1feaa78c9868: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-target + name: gpt-3-cs-en-target reference: GPT-3 paper f25d54ca-cee7-4e06-b67f-6be82672707d: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-target + name: xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/de-en/templates.yaml b/promptsource/templates/wmt16/de-en/templates.yaml index 62e81ec72..5ed7065ad 100644 --- a/promptsource/templates/wmt16/de-en/templates.yaml +++ b/promptsource/templates/wmt16/de-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' 2f8a0523-7036-4696-bb89-bf5d69f5993e: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 30177c30-223b-4919-850e-744a32606d6a: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 388eff35-8559-406c-a733-14d2ad09dd52: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' 38a01751-b14a-42bb-896f-29ad7ea993f3: !Template answer_choices: null @@ -67,8 +63,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' 4063ab3f-5467-4842-b445-a65e67c6c7e0: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 42e7588b-a31d-4e83-a09d-642a943ca4e2: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper 5476dadf-95bf-4978-82c4-bf30708f00b5: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' 66bab156-55dd-4675-83ec-a70d5bf9dd9d: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' 6c3bd7a7-3c2c-47e9-b9da-f760d3b9ee02: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' 6f6c002f-4f20-42a1-ad4c-2d34aa892c75: !Template answer_choices: null @@ -152,8 +142,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper 87f1f54f-ff9e-4eae-94db-9d260fe1965a: !Template answer_choices: null @@ -166,8 +155,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' 89f344f9-869d-41c9-a647-1adadbb9e7e1: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' 95e7c43a-e477-410c-a50f-b64cef3ed1e9: !Template answer_choices: null @@ -194,8 +181,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper 995c6ce7-020e-40d1-a30f-9a526e1f841c: !Template answer_choices: null @@ -208,8 +194,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' a3d29f52-0a5c-4250-8fb4-b36009602c9e: !Template answer_choices: null @@ -223,8 +208,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' a3e46f64-557e-4fb0-94d6-e0961b7b2fb7: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf b07c1795-16c5-4389-b6e9-8292a8bc54d6: !Template answer_choices: null @@ -251,8 +234,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 c47dfcfd-1d10-4744-875a-2a11eb28fa17: !Template answer_choices: null @@ -265,8 +247,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 c5be4a28-f5cf-4444-80c0-0c67bcc283d6: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' c9ec611f-ec0e-4c97-ad56-71f3aeef5e41: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper d45b599a-2f9d-4d56-a7cf-da02ac2a16e4: !Template answer_choices: null @@ -307,8 +286,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' e7232521-1517-44ec-8a51-85f5db80c1eb: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' e99b37c7-6d10-4a14-a462-8b91f2ef0f7e: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' f580b5fc-4f9c-4b8e-b979-028475218e72: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' fb2fc0a0-7f82-4bfb-a4a8-d38ed93400aa: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' diff --git a/promptsource/templates/wmt16/fi-en/templates.yaml b/promptsource/templates/wmt16/fi-en/templates.yaml index 4e0abaf76..1dcc51907 100644 --- a/promptsource/templates/wmt16/fi-en/templates.yaml +++ b/promptsource/templates/wmt16/fi-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-target + name: gpt-3-en-fi-target reference: GPT-3 paper 11c13b7b-ca3b-492d-ac9c-047046969ed7: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-source+target + name: a_good_translation-fi-en-source+target reference: '' 21ea1113-61f4-4944-a844-162345673f80: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-source+starget + name: translate-en-fi-source+starget reference: '' 2913d203-ac99-4b85-a0d9-57157ab002e9: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-source+target + name: translate_as_en-fi-source+target reference: '' 360be0fb-aef3-4be7-974c-0d1e68f678ec: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-target + name: a_good_translation-fi-en-target reference: '' 40fc37c3-33dd-45a9-96ff-a4a1a77f8008: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-target + name: version-en-fi-target reference: '' 42ad8580-10ba-4df3-8efb-5c531db58279: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-target + name: translate_as_en-fi-target reference: '' 48593d42-382b-4ee0-9db9-a9323e7f20f1: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-source-target + name: xglm-en-fi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 500e6c30-7467-4ba7-8444-e3760fd624ad: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-target + name: translate-en-fi-target reference: '' 746854b6-6ad3-494b-818b-33424c4f5e0f: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fi-en-target + name: how_to_say-fi-en-target reference: '' 83e27a76-96d8-41f9-a128-93a43a66c5ab: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-target + name: translate_as_fi-en-target reference: '' 8f218d6c-ed80-421a-8714-b2bf8195982f: !Template answer_choices: null @@ -166,8 +155,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-target + name: a_good_translation-en-fi-target reference: '' 9728f940-4009-4772-bed0-67afe20b94c7: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-source+target + name: xglm-fi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 9aa17975-412f-4f0c-9aab-0a2dfcc1289d: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-source+target + name: a_good_translation-en-fi-source+target reference: '' a35aecd2-f51f-4975-8653-6b983d1bb641: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-target + name: translate-fi-en-target reference: '' a79f0128-f6fb-49b3-a27d-7883c17604d9: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-target + name: version-fi-en-target reference: '' acb870da-f2d4-4b8c-b445-05424f90a81e: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-target + name: xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 b1676638-561d-4bbf-9f44-4626b74a468e: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-source+target + name: gpt-3-en-fi-source+target reference: GPT 3 paper b9ab9732-c8ff-479c-9a4d-d70b6776c2a1: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-target + name: gpt-3-fi-en-target reference: GPT-3 paper c196ea94-5087-4312-9711-75e3933a2f00: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-source+target + name: version-fi-en-source+target reference: '' c365ebd2-2981-44ae-8291-b36550bffecb: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-source+target + name: translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf c9d0c7a8-89ee-48b0-9c24-6dbb1dd8025f: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-source+target + name: translate_as_fi-en-source+target reference: '' d83bbd3f-e0d3-41a5-a16e-62f2995f485b: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-source+target + name: gpt-3-fi-en-source+target reference: GPT-3 paper d9dfea8a-2f5a-47f6-90e0-7c6875c5f027: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fi-target + name: how_to_say-en-fi-target reference: '' efacbe75-c4ff-4dd9-bbfa-80f4bee92f60: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-source+target + name: version-en-fi-source+target reference: '' f8b963ef-3189-40d2-9eee-75b7285c3fdd: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-target + name: xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/ro-en/templates.yaml b/promptsource/templates/wmt16/ro-en/templates.yaml index 388cf9144..bddbf812c 100644 --- a/promptsource/templates/wmt16/ro-en/templates.yaml +++ b/promptsource/templates/wmt16/ro-en/templates.yaml @@ -11,8 +11,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ro-en-source+target + name: a_good_translation-ro-en-source+target reference: '' 10043255-0631-41e6-9f93-20f864d69eeb: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ro-en-source+target + name: xglm-ro-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 113e022f-9d70-40f1-855f-024c5badad20: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ro-en-source+target + name: translate-ro-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 1519ffc7-7a0b-4a04-b626-7265f93ad09f: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ro-target + name: version-en-ro-target reference: '' 1709e27a-1069-4124-9ab9-fb571d05cd0b: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ro-source+starget + name: translate-en-ro-source+starget reference: '' 22ecf1d7-6909-4b18-aac1-42d4104a101e: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ro-en-target + name: xglm-ro-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 26818653-4a13-456a-919e-8e46f4cca807: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ro-target + name: how_to_say-en-ro-target reference: '' 29397cea-e7fe-46c8-821d-75925174235b: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ro-source+target + name: a_good_translation-en-ro-source+target reference: '' 305a6ff4-5f27-4150-96cb-0f4657065de6: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ro-en-target + name: gpt-3-ro-en-target reference: GPT-3 paper 3ea8048e-8c88-47cf-85e1-be83a925edde: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ro-target + name: gpt-3-en-ro-target reference: GPT-3 paper 53283c05-89bd-40af-b68a-588e26bcabf4: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ro-en-source+target + name: translate_as_ro-en-source+target reference: '' 70758f4b-f327-4490-95fa-0998502e83dc: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ro-target + name: translate_as_en-ro-target reference: '' 7d6958fb-c088-49f3-a7a0-06f031045413: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ro-en-target + name: a_good_translation-ro-en-target reference: '' 8307a62a-4fe3-4c51-bd6a-39aa0ad1869e: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ro-target + name: a_good_translation-en-ro-target reference: '' 8391bcb6-6643-4cec-ad86-218820783c56: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ro-source-target + name: xglm-en-ro-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 8e245134-0607-4fa0-abbf-7725cfce1b7c: !Template answer_choices: null @@ -223,8 +208,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ro-target + name: translate-en-ro-target reference: '' 97592b70-84ec-4571-8101-844ee3bffac8: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ro-target + name: xglm-en-ro-target reference: XGLM paper https://arxiv.org/abs/2112.10668 a438d7f1-3421-4c58-a211-2d33c7c393cf: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ro-en-source+target + name: version-ro-en-source+target reference: '' a5604743-a810-4ae3-9811-f3906d7fcebb: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ro-en-target + name: translate_as_ro-en-target reference: '' b7c7bbe1-e873-4a5b-b86b-1689cf814cc9: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ro-en-target + name: translate-ro-en-target reference: '' d13385f1-2dfa-486d-9d64-8fcca0c8f449: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ro-source+target + name: gpt-3-en-ro-source+target reference: GPT 3 paper e3b3b252-d557-4640-b21a-df27fe7a03ce: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ro-en-source+target + name: gpt-3-ro-en-source+target reference: GPT-3 paper e83b3367-9a91-41db-a161-ddfdc30cfca1: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ro-source+target + name: translate_as_en-ro-source+target reference: '' f317bb24-5828-43d8-8f8f-74b193f96396: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ro-source+target + name: version-en-ro-source+target reference: '' f70cde91-976b-44d3-a77b-284e26aba970: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ro-en-target + name: version-ro-en-target reference: '' fd8c924a-93bb-480f-be25-94ac888b7e30: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ro-en-target + name: how_to_say-ro-en-target reference: '' diff --git a/promptsource/templates/wmt16/ru-en/templates.yaml b/promptsource/templates/wmt16/ru-en/templates.yaml index 3dc589867..142115b40 100644 --- a/promptsource/templates/wmt16/ru-en/templates.yaml +++ b/promptsource/templates/wmt16/ru-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' 09094227-68fe-4462-9477-11683a34452f: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 094e53f2-faec-40df-bbce-43baf9aab2bb: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 0beb1dab-caed-4fdf-b248-aa8d1013c09a: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' 232d5901-cbeb-4db3-b714-56bd2afd30a1: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' 232f4031-3ef7-4929-9d7c-f248ac0aef6c: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper 31503446-f6ce-48e4-8f12-4f7a9ba34e38: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 34fcf6e1-326b-48db-81da-b8a545991c6b: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 471fb738-1a7d-4f2c-a65b-71a78dc8f8cf: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' 59cffc19-7a33-44cb-a8a6-a29f61ae000e: !Template answer_choices: null @@ -141,8 +132,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' 5b4822ef-15c1-446e-85c7-b6665c96feb3: !Template answer_choices: null @@ -156,8 +146,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' 69bd54a9-2ec4-4c30-b951-4ce9875e4135: !Template answer_choices: null @@ -170,8 +159,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' 6f97021a-7679-4afb-b60c-454c5951b3ca: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' 6fe335f8-c91a-41b6-8a31-1cfe03e19f7b: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 87c91692-c5d2-46d7-b700-4238f7156e05: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper 90adb529-bc50-42d1-8229-bdfaeb52cf68: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 9aa8bab8-460c-44a4-9367-60d2d32be39d: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' 9c49e553-4823-46b8-9bfe-6b29761dc88f: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' a55c925c-d1e0-4d86-95cf-e0d9477d1e86: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' aed6444f-0558-4234-8b2a-6885a5906491: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper b02c260c-50ce-40d6-b9eb-8f8c69285cf6: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' b56e3f29-475c-4b2a-ab2b-1ca9de77a538: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' b844d53a-20dd-4430-b47a-cfd759a7cafb: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' e2c7cce8-1955-4fea-9a9a-9e728d9c37f9: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' fcf79f35-fd25-4281-ab6e-d1619d2477a7: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' febf5e43-6af8-4d1e-8c20-3e454239c420: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt16/tr-en/templates.yaml b/promptsource/templates/wmt16/tr-en/templates.yaml index cf149420c..0d4d1db5d 100644 --- a/promptsource/templates/wmt16/tr-en/templates.yaml +++ b/promptsource/templates/wmt16/tr-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-target + name: xglm-en-tr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 0a3d5810-f12a-4bf4-8f73-796445516308: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-target + name: a_good_translation-en-tr-target reference: '' 2798539b-6159-4cfe-ad24-26adcd0de104: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-source+target + name: gpt-3-tr-en-source+target reference: GPT-3 paper 32814e82-f25c-4421-827a-801c37507852: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-target + name: translate-tr-en-target reference: '' 3d717faf-57ea-4912-8441-93e8227fbe66: !Template answer_choices: null @@ -67,8 +63,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-source-target + name: xglm-en-tr-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 3d8a7fef-1a66-499c-83aa-2729372dc315: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-target + name: version-en-tr-target reference: '' 4ecce376-edea-4750-82ed-d47227133584: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-tr-en-target + name: how_to_say-tr-en-target reference: '' 529393ed-0962-43de-8031-422e8cc8cdd9: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-target + name: a_good_translation-tr-en-target reference: '' 5d439e41-e34b-4dad-b883-0228054dfc75: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-source+starget + name: translate-en-tr-source+starget reference: '' 81fd39b4-471b-46fc-9171-73b0ebfaeff3: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-source+target + name: translate_as_en-tr-source+target reference: '' 81ffb065-63b0-42d0-abc7-d0b4af2eae6b: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-target + name: gpt-3-en-tr-target reference: GPT-3 paper 820db088-1fb4-4cab-b47b-447b8958b108: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-source+target + name: xglm-tr-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 85e5c442-bd52-41e1-810f-039c9d30778f: !Template answer_choices: null @@ -181,8 +169,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-source+target + name: gpt-3-en-tr-source+target reference: GPT 3 paper 936cef42-229d-40f1-8c1c-1c077f2e6e48: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-target + name: translate_as_en-tr-target reference: '' 9eb17600-3717-44c6-90e6-103bd40e16d8: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-target + name: version-tr-en-target reference: '' a028ed7a-482b-4ac9-9339-a888c255e09f: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-source+target + name: a_good_translation-en-tr-source+target reference: '' a335c2d1-09a5-43fc-91c5-62de6fdde9f1: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-source+target + name: version-en-tr-source+target reference: '' aeae66d6-b0b6-471b-af3c-51a9d3100b52: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-target + name: xglm-tr-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 b3fe56dc-1a02-4b1b-a0bc-58a4e4e3118e: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-source+target + name: translate-tr-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf c7f4214d-6581-4078-99c1-50eba4b06f97: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-source+target + name: translate_as_tr-en-source+target reference: '' cfbad6dc-82f1-4f07-be6b-71e694d479a4: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-source+target + name: version-tr-en-source+target reference: '' d37ce743-96e7-4a0d-8dad-d51567bc3904: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-source+target + name: a_good_translation-tr-en-source+target reference: '' e48579b1-ca64-4434-a8f7-232c9b08abee: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-target + name: gpt-3-tr-en-target reference: GPT-3 paper e7d0083f-4003-4e4f-b78c-737d47dd0822: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-target + name: translate_as_tr-en-target reference: '' ea16adca-5a2e-4d47-aaa5-73b8c16d7409: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-tr-target + name: how_to_say-en-tr-target reference: '' fb6a6dba-9d0a-4475-9899-8e9567bbe3be: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-target + name: translate-en-tr-target reference: '' diff --git a/promptsource/templates/wmt17/cs-en/templates.yaml b/promptsource/templates/wmt17/cs-en/templates.yaml index f9e7cf2e1..d58f1c095 100644 --- a/promptsource/templates/wmt17/cs-en/templates.yaml +++ b/promptsource/templates/wmt17/cs-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-source+target + name: translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 1504c2ef-f956-42b1-b6d3-58bc82859f8c: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-source+target + name: translate_as_cs-en-source+target reference: '' 1b44c745-8efe-4aa3-9a78-b4bde3d0e20b: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-source+target + name: a_good_translation-cs-en-source+target reference: '' 2d15416d-0be5-4531-9506-cdbbb1fe8e5b: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-source+target + name: gpt-3-cs-en-source+target reference: GPT-3 paper 2e3bfc96-c1e1-4180-8cc4-9dfae95d00b6: !Template answer_choices: null @@ -66,8 +62,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-target + name: a_good_translation-en-cs-target reference: '' 30898b95-7fcc-44fe-b41d-2fccd9247d52: !Template answer_choices: null @@ -80,8 +75,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-target + name: xglm-cs-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 3d8622c7-25b5-4a7d-a156-5f6ac289bc64: !Template answer_choices: null @@ -94,8 +88,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-source+target + name: xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 4a609be1-08a1-41f8-a2b4-d166107e1025: !Template answer_choices: null @@ -108,8 +101,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-target + name: translate-cs-en-target reference: '' 50904822-5371-4cd0-9c3c-27bcda65ad93: !Template answer_choices: null @@ -122,8 +114,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-source+starget + name: translate-en-cs-source+starget reference: '' 516bdcfa-8f19-4d91-b839-723987de3cc8: !Template answer_choices: null @@ -136,8 +127,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-target + name: translate_as_cs-en-target reference: '' 53af174b-3bd1-41fe-aff5-c72963a8d7d1: !Template answer_choices: null @@ -150,8 +140,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-target + name: gpt-3-en-cs-target reference: GPT-3 paper 582fa43b-c399-4e47-bf65-3f6eaac4c2a6: !Template answer_choices: null @@ -165,8 +154,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-target + name: a_good_translation-cs-en-target reference: '' 73618299-da46-46a8-992c-4d8a9d178aca: !Template answer_choices: null @@ -179,8 +167,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-target + name: xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 843b5407-062b-4135-9930-8bfff41f0500: !Template answer_choices: null @@ -193,8 +180,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-source-target + name: xglm-en-cs-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 914d4d86-9465-4a70-b5ba-6907c61811f6: !Template answer_choices: null @@ -207,8 +193,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-target + name: translate_as_en-cs-target reference: '' a37d598e-8b5c-4cd4-8579-acd131cdf842: !Template answer_choices: null @@ -222,8 +207,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-target + name: version-cs-en-target reference: '' a7f45664-c9a5-436d-832e-1462094d6402: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-source+target + name: version-en-cs-source+target reference: '' a9af3ff9-fadb-49d8-95d0-be53093b27e1: !Template answer_choices: null @@ -251,8 +234,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-target + name: translate-en-cs-target reference: '' aa7740dd-b61f-4146-a020-8c5bbbf0d287: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-source+target + name: a_good_translation-en-cs-source+target reference: '' ccf5c2ab-7a25-4b96-96c9-040fcb6ec9db: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-source+target + name: translate_as_en-cs-source+target reference: '' d54087bc-3a16-4a8e-ad86-e1fe067d6b6d: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-cs-target + name: how_to_say-en-cs-target reference: '' d91a9f47-a8cd-4731-9a3d-d6d5b547014d: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-target + name: version-en-cs-target reference: '' e6200253-6ce1-4179-a8a0-ed7fd9714e62: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-source+target + name: version-cs-en-source+target reference: '' ec893b3e-435c-4557-9c7f-c3c8c2e42a64: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-cs-en-target + name: how_to_say-cs-en-target reference: '' f2ac2dab-2f82-4b42-a03d-0a4ff8a90549: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-target + name: gpt-3-cs-en-target reference: GPT-3 paper fd1edae5-aa00-4e4c-8941-6ade010da2f0: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-source+target + name: gpt-3-en-cs-source+target reference: GPT 3 paper diff --git a/promptsource/templates/wmt17/de-en/templates.yaml b/promptsource/templates/wmt17/de-en/templates.yaml index 454089bf2..8087f3d75 100644 --- a/promptsource/templates/wmt17/de-en/templates.yaml +++ b/promptsource/templates/wmt17/de-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' 0c9f2ead-1805-4af4-8f68-641946ead546: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' 0de0d489-36bb-4d6e-8e3f-57973cee1461: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper 199cba3c-a7e7-4205-b681-81dab0fd81c2: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' 20a1dc77-2e87-4a96-a515-babf03a165e6: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 37b0220b-f12d-48ba-bdc1-af0493b011fb: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' 3a23ebf6-31b6-4cbb-a0a8-7872560fd8da: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' 44b0f1c2-c91f-4639-9d4a-2991e7001501: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' 4513e0e4-7122-4cb1-8d35-8893cbd276d6: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' 6457e3da-34e6-4a9e-b1d5-29006e67478a: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper 65b06f84-3937-4d5f-9b07-a97135391b07: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' 6e84bea7-ed10-43bc-a80b-fbfeba40fec0: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' 6f76439a-5335-4880-8072-0c02671c3b68: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 79613dca-7808-4ada-9f2b-5502f4201001: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper 86d73059-130f-42e4-a765-e2d53ef72de1: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 8882e270-2797-4d00-9061-3da89ebb0c15: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' 8be79e83-710b-4669-b769-e5669c63831f: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper 97584294-7bb0-4434-aa64-eea905210e2a: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' a46ff7ba-7890-4da2-85fd-ca8b8fc54dda: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' afb88539-be1a-40e7-8b1e-8b4d9b035cc3: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 d1f4b6ac-e930-43ec-8970-c95adcf4b2af: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 d6f06333-44b3-45fe-9161-8f26af62e8c6: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' d72bb3dd-48de-47cb-97a5-bdf03d8246f7: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' dc567cdc-14ec-4b86-8282-2e4cfe9cc45b: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 ea5b141a-0097-4aa4-9ab5-e61e7e691fbe: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' ebec0874-7b1d-4191-848c-d55d5c6f10fd: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' diff --git a/promptsource/templates/wmt17/fi-en/templates.yaml b/promptsource/templates/wmt17/fi-en/templates.yaml index d37368c08..87e10308e 100644 --- a/promptsource/templates/wmt17/fi-en/templates.yaml +++ b/promptsource/templates/wmt17/fi-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-target + name: translate-fi-en-target reference: '' 0d23b84d-10f9-43f0-9972-c4f1273d0230: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-source+target + name: gpt-3-en-fi-source+target reference: GPT 3 paper 164c33b9-a1c2-457a-b8e6-fb4e03a4ac14: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-target + name: xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 1ab09eb2-48c9-4f48-96ea-d3bf8875a363: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-source+target + name: xglm-fi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 1d028db9-5f27-4ff6-a9bf-3b8f7e8f49eb: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-target + name: translate-en-fi-target reference: '' 1fea6184-a880-4f5a-9856-ad825bb2ad86: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-source+target + name: gpt-3-fi-en-source+target reference: GPT-3 paper 22a056a5-9ac1-4f87-83df-3ae8326b4693: !Template answer_choices: null @@ -95,8 +89,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-target + name: a_good_translation-en-fi-target reference: '' 281fb66a-d0f2-40a7-a6e8-4c5844bed10c: !Template answer_choices: null @@ -109,8 +102,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-target + name: translate_as_en-fi-target reference: '' 3b679f65-aa5b-4bc2-bb47-b5dada7fb0f6: !Template answer_choices: null @@ -123,8 +115,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fi-target + name: how_to_say-en-fi-target reference: '' 3d1b6cef-c40e-4965-ad0b-2795b15a3f45: !Template answer_choices: null @@ -137,8 +128,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-target + name: gpt-3-fi-en-target reference: GPT-3 paper 50d7b5e4-8781-4f74-b710-ac0e7b14a763: !Template answer_choices: null @@ -151,8 +141,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-target + name: gpt-3-en-fi-target reference: GPT-3 paper 5c9db42a-4000-422d-ba74-8cec1f12cf3e: !Template answer_choices: null @@ -165,8 +154,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-source+target + name: translate_as_fi-en-source+target reference: '' 5d973ade-560b-4813-9c76-a5a1e415da85: !Template answer_choices: null @@ -179,8 +167,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-source+target + name: translate_as_en-fi-source+target reference: '' 67bc9eb3-0769-4357-8b2f-15b8c3b9abf7: !Template answer_choices: null @@ -193,8 +180,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-source+starget + name: translate-en-fi-source+starget reference: '' 8053805e-fc9d-43e8-82d4-e30e3a76509a: !Template answer_choices: null @@ -207,8 +193,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fi-en-target + name: how_to_say-fi-en-target reference: '' 882f5ee3-d108-4811-9876-0972b437e45e: !Template answer_choices: null @@ -221,8 +206,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-source+target + name: translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 92ecb048-a030-427f-96c7-86732efe4245: !Template answer_choices: null @@ -235,8 +219,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-source-target + name: xglm-en-fi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 a788c720-369e-439f-8805-4a18530044d7: !Template answer_choices: null @@ -250,8 +233,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-target + name: version-en-fi-target reference: '' b12a7f30-947c-4ee6-80ae-ff439beaab15: !Template answer_choices: null @@ -263,8 +245,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-source+target + name: a_good_translation-fi-en-source+target reference: '' b3dd87d1-9c83-4350-b0e2-bf1a78222aa7: !Template answer_choices: null @@ -277,8 +258,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-target + name: xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 ceddd576-ee41-4e4d-8b9d-e1ecb9bfcc68: !Template answer_choices: null @@ -292,8 +272,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-target + name: version-fi-en-target reference: '' d031d8ac-0fa0-40ba-bb61-b9da287bb8cf: !Template answer_choices: null @@ -307,8 +286,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-source+target + name: version-en-fi-source+target reference: '' d2e78da8-9a4b-499a-b7fa-5802fd1a5fdd: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-source+target + name: a_good_translation-en-fi-source+target reference: '' d42a1dda-15e1-4630-be21-4de22e03dcec: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-target + name: a_good_translation-fi-en-target reference: '' e8db7580-1543-4088-9cdb-f75879490e6c: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-source+target + name: version-fi-en-source+target reference: '' f8a9585e-dd83-4941-ada1-2125b81637d6: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-target + name: translate_as_fi-en-target reference: '' diff --git a/promptsource/templates/wmt17/lv-en/templates.yaml b/promptsource/templates/wmt17/lv-en/templates.yaml index d6a7c6cea..36d9bd3be 100644 --- a/promptsource/templates/wmt17/lv-en/templates.yaml +++ b/promptsource/templates/wmt17/lv-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-lv-en-target + name: a_good_translation-lv-en-target reference: '' 0936fbe5-6138-437d-9ccc-44d02545d02e: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-lv-en-target + name: translate-lv-en-target reference: '' 09e7d1ab-2efb-4130-a68c-252c483e6d44: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-lv-en-target + name: xglm-lv-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 1b20a572-10df-47ad-8318-b7d367de2a06: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-lv-target + name: gpt-3-en-lv-target reference: GPT-3 paper 1c127357-54d2-434e-a346-c6116b7ae131: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-lv-source+target + name: gpt-3-en-lv-source+target reference: GPT 3 paper 1dddcbde-f11c-44bd-a6dc-abc14c9ff8d2: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-lv-source+target + name: translate_as_en-lv-source+target reference: '' 2201a3af-bb28-4cb7-afc0-dfeecef0def2: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-lv-en-target + name: gpt-3-lv-en-target reference: GPT-3 paper 25e58c80-4a60-4477-b406-cf7d9cc6dd57: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-lv-en-source+target + name: a_good_translation-lv-en-source+target reference: '' 2b168e9b-b592-4ef8-aff0-74a06059526f: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_lv-en-target + name: translate_as_lv-en-target reference: '' 3a396647-7b0e-4d4c-8e9c-454343a532af: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-lv-en-source+target + name: xglm-lv-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 442985e4-d28c-4814-aa88-51582ea4bd8a: !Template answer_choices: null @@ -152,8 +142,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-lv-en-target + name: how_to_say-lv-en-target reference: '' 4630798f-5bff-412d-bac3-1eb26977cb43: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-lv-en-source+target + name: version-lv-en-source+target reference: '' 4a89beac-0b24-4929-a60b-86e67bdf89d1: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-lv-target + name: version-en-lv-target reference: '' 5159f3c9-4cef-4e23-b0c6-101d27d1a561: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_lv-en-source+target + name: translate_as_lv-en-source+target reference: '' 5358c036-b6b0-48ad-a8fb-8296c35a53ea: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-lv-en-source+target + name: gpt-3-lv-en-source+target reference: GPT-3 paper 5c6c9aec-1736-448a-9036-f0f5436838cc: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-lv-source+starget + name: translate-en-lv-source+starget reference: '' 79f2149d-78fe-43f0-bb7e-aadc75d5b045: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-lv-target + name: xglm-en-lv-target reference: XGLM paper https://arxiv.org/abs/2112.10668 854685ec-5e2e-46f7-b47e-a3a5502179b5: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-lv-target + name: translate_as_en-lv-target reference: '' 87b88026-d3c8-4b36-b6fc-3754883ce195: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-lv-source+target + name: version-en-lv-source+target reference: '' 912ea72b-7973-44bc-afb8-4635fd55a838: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-lv-target + name: how_to_say-en-lv-target reference: '' a7605476-3722-4b9c-80c3-f1fa063d63f5: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-lv-source+target + name: a_good_translation-en-lv-source+target reference: '' c7733bca-74bd-478b-b4a0-ae399b8ee720: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-lv-en-source+target + name: translate-lv-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf d3d7c0cf-57bf-4067-9229-529ea7a520f5: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-lv-source-target + name: xglm-en-lv-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 f2924798-0423-43f9-b203-251dacec9a7a: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-lv-target + name: translate-en-lv-target reference: '' f32085fc-eb1b-4383-88b7-e9eea44fa379: !Template answer_choices: null @@ -353,8 +329,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-lv-en-target + name: version-lv-en-target reference: '' fa246f5b-3a3d-44a1-9f3e-502572ead9a6: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-lv-target + name: a_good_translation-en-lv-target reference: '' diff --git a/promptsource/templates/wmt17/ru-en/templates.yaml b/promptsource/templates/wmt17/ru-en/templates.yaml index 523cd3b1f..2429c8ae2 100644 --- a/promptsource/templates/wmt17/ru-en/templates.yaml +++ b/promptsource/templates/wmt17/ru-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' 0f53aa20-a3ff-4dcc-8178-0ff8563491bd: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' 1cb0dfa5-cac2-4a05-aa0e-48a4796f8a81: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' 2274b7e4-9d86-4f1c-a945-604deeae3e8c: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' 2babd6e7-8d10-4eb9-93be-7d27b3e675b9: !Template answer_choices: null @@ -70,8 +66,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' 31b8fd81-e2f9-4a1a-827d-efd85356ba09: !Template answer_choices: null @@ -85,8 +80,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' 352474e6-f128-40e6-ba92-2d9b5d259796: !Template answer_choices: null @@ -99,8 +93,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 3fbb2028-f9b2-45dd-bc56-c77fdf27bb67: !Template answer_choices: null @@ -113,8 +106,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 5dd35a60-5609-40bb-873e-4a3052be1dfc: !Template answer_choices: null @@ -128,8 +120,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 66bf33ae-0cf8-4649-8ec8-40fcb4ab2e8f: !Template answer_choices: null @@ -142,8 +133,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' 6b05359f-8d2e-467b-ab2f-2bfafd9020a3: !Template answer_choices: null @@ -156,8 +146,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper 7489148f-3d36-4bfb-831b-6dc776e09d0a: !Template answer_choices: null @@ -170,8 +159,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper 79e2e60d-2905-4718-80c4-702b880bed86: !Template answer_choices: null @@ -184,8 +172,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper 7a2332c9-4e93-4338-90f7-32dd73a13454: !Template answer_choices: null @@ -198,8 +185,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' 8552f282-7ec0-43f1-9055-afc034676c3f: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' 9150eacb-0841-45ac-961d-6a95856c5803: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' a01d797c-29ff-45bc-b33f-1833692ae75b: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' a380ddce-9937-4235-87cf-d2e76658f590: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 ac8b312d-f909-4409-8453-3430b88c6bae: !Template answer_choices: null @@ -269,8 +251,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' c6e3beed-6b46-487f-84e6-4621689fa107: !Template answer_choices: null @@ -283,8 +264,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' db9616b7-d2d0-434a-90ee-26b8e27263dc: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' df059410-48f4-4282-84dc-5d0724d63b25: !Template answer_choices: null @@ -311,8 +290,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 e40de616-dc50-44f4-859c-276a556c00a0: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 eba2ae8b-ec7a-40c8-bb96-c74d8755cc03: !Template answer_choices: null @@ -339,8 +316,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf ef94f42c-dcab-40e0-9651-7e471a504320: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' f200f945-6201-468b-b675-a6660f152c8f: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' diff --git a/promptsource/templates/wmt17/tr-en/templates.yaml b/promptsource/templates/wmt17/tr-en/templates.yaml index e004575e9..0a1ade384 100644 --- a/promptsource/templates/wmt17/tr-en/templates.yaml +++ b/promptsource/templates/wmt17/tr-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-target + name: translate_as_en-tr-target reference: '' 13feec66-ea23-485c-833e-e75c1aa39c6b: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-tr-target + name: how_to_say-en-tr-target reference: '' 215d9910-906f-4e27-91a9-d4e2b10ca8c5: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-source+target + name: version-tr-en-source+target reference: '' 2e3c2cbb-0069-44b0-8c5f-1dba0f82e52f: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-source+target + name: xglm-tr-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 3b6a8171-c577-46c9-a6c6-71f06347c296: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-tr-en-target + name: how_to_say-tr-en-target reference: '' 4508bd9c-1d22-4a29-bb03-270646863f54: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-target + name: gpt-3-tr-en-target reference: GPT-3 paper 57019450-25a1-4af5-9264-c7c75f4426c1: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-target + name: translate-tr-en-target reference: '' 57f1aa08-5e65-48f2-9f2b-153ab8313120: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-target + name: translate-en-tr-target reference: '' 61123501-a3ff-469b-a952-11fd3a5ff6c5: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-target + name: xglm-tr-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 6190b0e7-e1c5-48bd-8f1c-13634b044f69: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-target + name: xglm-en-tr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 709c77c1-7710-4e53-ad0d-4ac623221e26: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-source+target + name: version-en-tr-source+target reference: '' 71a32989-8fd0-446f-b17e-eea66875a507: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-source+target + name: a_good_translation-en-tr-source+target reference: '' 73623593-9c11-4348-8233-df90762b2e20: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-target + name: gpt-3-en-tr-target reference: GPT-3 paper 7954790f-6fb4-4edb-b862-0549b3b22e58: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-source-target + name: xglm-en-tr-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 7a290d8f-87d0-480f-a65d-9e52c11d3fe6: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-target + name: version-tr-en-target reference: '' 7c41a569-cd15-4d0f-b2c5-96146e853316: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-source+target + name: a_good_translation-tr-en-source+target reference: '' 9391038a-e0ed-49e0-8bbb-814b9afef7bc: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-source+target + name: gpt-3-en-tr-source+target reference: GPT 3 paper 9e7bcfbc-73d6-41b5-9b9c-51d4be2455ed: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-target + name: a_good_translation-tr-en-target reference: '' ad102733-eee8-4ad7-9bb0-8a0629e02935: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-source+target + name: translate_as_en-tr-source+target reference: '' b0d7f2ab-fdf6-43ef-a34e-61778d68f2e7: !Template answer_choices: null @@ -283,8 +264,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-target + name: version-en-tr-target reference: '' b554789a-45c1-4713-9788-c5e9789e36d0: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-source+starget + name: translate-en-tr-source+starget reference: '' b9272dc4-5f6e-462a-8c49-9bacb4843e0b: !Template answer_choices: null @@ -311,8 +290,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-source+target + name: translate-tr-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf bee2e383-cc8e-43a1-a317-ba7904a2b70a: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-target + name: a_good_translation-en-tr-target reference: '' e23d2cf8-e1b0-4305-8326-ecd758a792e8: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-source+target + name: translate_as_tr-en-source+target reference: '' f4601284-1c19-4c50-98d1-d1be4ef142ce: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-source+target + name: gpt-3-tr-en-source+target reference: GPT-3 paper fe200cad-1f06-4805-943a-88355d1ce6e2: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-target + name: translate_as_tr-en-target reference: '' diff --git a/promptsource/templates/wmt17/zh-en/templates.yaml b/promptsource/templates/wmt17/zh-en/templates.yaml index 14a99458f..f83f498e1 100644 --- a/promptsource/templates/wmt17/zh-en/templates.yaml +++ b/promptsource/templates/wmt17/zh-en/templates.yaml @@ -11,8 +11,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-source+target + name: a_good_translation-zh-en-source+target reference: '' 1022f169-5b09-4efc-bb8c-c80b1476c6ea: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-source-target + name: xglm-en-zh-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 24c02b50-8aec-4645-91ba-e6b74b4659ce: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-source+target + name: version-en-zh-source+target reference: '' 2ef8654a-9cce-4602-88f5-f56f1b8d5d88: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-source+starget + name: translate-en-zh-source+starget reference: '' 38349a24-8617-4ce2-ac6b-6e399fa5f926: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-source+target + name: version-zh-en-source+target reference: '' 3a8705c6-a21d-47e3-917c-1eb50961af15: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-source+target + name: translate-zh-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 3b3240ec-ddc6-4f46-84c3-da784888db1c: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-target + name: translate_as_en-zh-target reference: '' 43f47446-52d2-41a3-9ca3-abbb3fc1a93f: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-source+target + name: xglm-zh-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 45a91ef1-00d9-474f-a945-c2acfaee245b: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-target + name: gpt-3-zh-en-target reference: GPT-3 paper 55efd980-0439-4eee-9ec9-bb8baf07bf8a: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-source+target + name: a_good_translation-en-zh-source+target reference: '' 6696796b-d19b-4cba-a840-18b234522e59: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-source+target + name: gpt-3-en-zh-source+target reference: GPT 3 paper 6b31d70d-c109-44a6-aae8-22918c41127d: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-target + name: xglm-en-zh-target reference: XGLM paper https://arxiv.org/abs/2112.10668 7c49b8a1-82cb-4ad3-96d2-be71e7bf4401: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-target + name: version-zh-en-target reference: '' 8b8653bb-d13a-4ad6-8236-d851b2036861: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-source+target + name: translate_as_zh-en-source+target reference: '' 909a8bb9-3c03-4a86-bd16-de054ce83f73: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-target + name: translate-en-zh-target reference: '' 9798947f-7a8e-4e1b-aa3a-8c51df5c761b: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-target + name: translate_as_zh-en-target reference: '' b4302b25-ec16-49e3-bbf5-f994b73c9108: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-target + name: a_good_translation-en-zh-target reference: '' b6b0428e-e988-4408-9cea-5de22a57006d: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-source+target + name: gpt-3-zh-en-source+target reference: GPT-3 paper cca404b5-2af7-493a-a31d-97c0417d1b28: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-zh-en-target + name: how_to_say-zh-en-target reference: '' d9987f06-c787-45d0-b05f-3b6857f2fcb7: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-target + name: xglm-zh-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 dd529993-6d65-497f-9cea-ba3104763099: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-target + name: gpt-3-en-zh-target reference: GPT-3 paper e2d3e03b-ee52-4819-a8f9-c97807ab15ae: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-zh-target + name: how_to_say-en-zh-target reference: '' e47899c3-8ec9-41a0-b78e-1379e8a04d61: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-target + name: a_good_translation-zh-en-target reference: '' f067f264-a065-466d-ba1a-90b88b6927bd: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-target + name: translate-zh-en-target reference: '' f0788fdb-f012-40d4-b5ff-f1a1e17472e8: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-source+target + name: translate_as_en-zh-source+target reference: '' f51d1d6f-0555-43fb-b00a-ae7800153d1a: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-target + name: version-en-zh-target reference: '' diff --git a/promptsource/templates/wmt18/cs-en/templates.yaml b/promptsource/templates/wmt18/cs-en/templates.yaml index 1b004aaeb..5d3e3a243 100644 --- a/promptsource/templates/wmt18/cs-en/templates.yaml +++ b/promptsource/templates/wmt18/cs-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-target + name: gpt-3-cs-en-target reference: GPT-3 paper 08968ba9-7655-4c6f-b944-15a79c3532cd: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-source+target + name: translate-cs-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 096afb1d-46ea-43b6-931e-2ec156616532: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-cs-en-target + name: how_to_say-cs-en-target reference: '' 0cd65c6f-88c2-47d9-890d-eebca06c04b5: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-source-target + name: xglm-en-cs-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 1437cd13-5350-4b01-8726-4dff81690098: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-target + name: a_good_translation-cs-en-target reference: '' 27666a03-bc94-49a8-a4e7-59f43402ac1f: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-source+target + name: translate_as_cs-en-source+target reference: '' 33725eb8-5f79-408f-9d7a-1fedb11d0aef: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-source+starget + name: translate-en-cs-source+starget reference: '' 34a73c3b-91ea-4995-b183-adf475db0006: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_cs-en-target + name: translate_as_cs-en-target reference: '' 3afe3c3a-8f80-4b2c-b0d4-1f53e288fc0e: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-cs-en-target + name: translate-cs-en-target reference: '' 4087a143-71a4-4174-89c2-af3836c3fbf1: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-source+target + name: xglm-cs-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 4a5edbb5-7647-48d5-8ffb-c99d39de35ab: !Template answer_choices: null @@ -152,8 +142,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-target + name: a_good_translation-en-cs-target reference: '' 6676a4d3-0621-43bb-bb97-269c3f3718f7: !Template answer_choices: null @@ -165,8 +154,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-cs-en-source+target + name: a_good_translation-cs-en-source+target reference: '' 7713da9d-ccdd-43ce-9895-503afee2bb8a: !Template answer_choices: null @@ -179,8 +167,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-cs-target + name: how_to_say-en-cs-target reference: '' 82200d10-5851-4f29-80f6-42392ab794a9: !Template answer_choices: null @@ -193,8 +180,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-target + name: gpt-3-en-cs-target reference: GPT-3 paper 838161af-dd29-463c-8561-f016f2f7054b: !Template answer_choices: null @@ -208,8 +194,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-target + name: version-cs-en-target reference: '' 91e38fee-b424-47bc-a8c3-b47eb4cf1624: !Template answer_choices: null @@ -222,8 +207,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-cs-target + name: translate-en-cs-target reference: '' 93d9e3cd-34ea-45f7-a2be-0452f7eec9dd: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-target + name: version-en-cs-target reference: '' 9cea0cbb-aa40-4e91-bdb9-3d926c647cbe: !Template answer_choices: null @@ -251,8 +234,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-cs-en-target + name: xglm-cs-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 a572a3b3-9ef4-482e-89b7-bc248231f75b: !Template answer_choices: null @@ -265,8 +247,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-source+target + name: translate_as_en-cs-source+target reference: '' a985c6f8-4a7f-4cba-a407-1931b048a3a3: !Template answer_choices: null @@ -279,8 +260,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-cs-en-source+target + name: gpt-3-cs-en-source+target reference: GPT-3 paper b3545af0-9c6e-49fa-beb3-40bcbda3128f: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-cs-source+target + name: version-en-cs-source+target reference: '' bbb3bf9c-ce66-490e-abbc-b3166600d813: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-cs-source+target + name: gpt-3-en-cs-source+target reference: GPT 3 paper c506feaa-fe44-4ddf-83c2-059e36811105: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-cs-target + name: translate_as_en-cs-target reference: '' cc4fc133-d6e1-4684-9bdd-b20fc22be63f: !Template answer_choices: null @@ -336,8 +313,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-cs-target + name: xglm-en-cs-target reference: XGLM paper https://arxiv.org/abs/2112.10668 ce04a4bb-7ea4-40b4-aac9-c268f9fecf3c: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-cs-en-source+target + name: version-cs-en-source+target reference: '' dce3d790-25d0-4b36-b47b-bc9935859cd8: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-cs-source+target + name: a_good_translation-en-cs-source+target reference: '' diff --git a/promptsource/templates/wmt18/de-en/templates.yaml b/promptsource/templates/wmt18/de-en/templates.yaml index ca3110abd..2c66c7c9a 100644 --- a/promptsource/templates/wmt18/de-en/templates.yaml +++ b/promptsource/templates/wmt18/de-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' 0d2188af-dc45-476b-9a6d-e3ecc53aa5cd: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' 12205792-1352-4ad8-a85a-7e3929022c8b: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' 28604275-67d1-4231-ac6a-73692dbc66c4: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' 48fff56c-0bec-4dbd-bf09-2a5e98a77219: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' 55c03738-d30d-4ce0-a3f9-ba548580351a: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' 5fa73a35-d47d-45ff-9457-6664066b01c4: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' 626b674e-c57d-452b-bfd9-6523520542e1: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper 67e8b71d-e37d-4336-a354-48c769f9a591: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 67f3ac0a-3ae1-4a7f-9a0e-0ef1619298aa: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' 70965178-8a49-47e7-8956-1c6d081eb090: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' 746f42a6-bdd3-4cae-b8da-5301f515c650: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper 75010174-babf-4c15-b21b-307f3ae1d739: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' 76fed237-5a46-4494-9d0c-c4158b046aa0: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 815753ac-61e0-4090-9298-b29e04244953: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 8d17685a-52c3-4600-a5f6-98f949e358bc: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' 8ed9dc8c-10ae-4744-8f80-c07a3b267602: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 91fe5fef-9f0a-496f-8bc0-f04043c1bc0f: !Template answer_choices: null @@ -255,8 +238,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' 9780d39d-4d81-4dfa-881a-7380f19a47fa: !Template answer_choices: null @@ -269,8 +251,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' b5866f82-9db7-4b89-9b6f-42168dcf4113: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' c0bf038f-845b-4e18-be58-cf8501cdd421: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper c30ea6e2-a77c-4372-8522-bc39a78cede5: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' c6e04cfa-dd7d-4623-93f8-179610625f48: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper ceace350-5d39-450b-bb2a-1c3187a19ed8: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf d4c41751-65b4-456f-b414-4ed886974c28: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 e7992f8f-a017-40b4-bb15-2ff043a4701b: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' diff --git a/promptsource/templates/wmt18/et-en/templates.yaml b/promptsource/templates/wmt18/et-en/templates.yaml index ad7155798..77504b3b0 100644 --- a/promptsource/templates/wmt18/et-en/templates.yaml +++ b/promptsource/templates/wmt18/et-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-et-source+target + name: a_good_translation-en-et-source+target reference: '' 0d0e6a01-1673-4296-9a17-d56ec1ed6983: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-et-target + name: a_good_translation-en-et-target reference: '' 0f639e5e-15c8-4526-96a5-03deae5754d2: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-et-target + name: translate_as_en-et-target reference: '' 1594d8db-e723-4451-8f14-62d933a70f27: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-et-en-source+target + name: a_good_translation-et-en-source+target reference: '' 1a003a8d-1fa6-4d49-b1bf-3510ab894be1: !Template answer_choices: null @@ -67,8 +63,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-et-source+target + name: gpt-3-en-et-source+target reference: GPT 3 paper 21144c2c-d45d-45e5-a1c1-0f8150fc0dc0: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-et-en-target + name: a_good_translation-et-en-target reference: '' 564e1318-7bf6-4e7b-9f9d-9ce31f33717d: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-et-en-target + name: version-et-en-target reference: '' 652947c9-9fe0-43b0-8645-f9b6f8cbc6b3: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-et-source+target + name: translate_as_en-et-source+target reference: '' 674e2912-7071-47a6-9dc2-e29420278f30: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-et-target + name: translate-en-et-target reference: '' 6b0f77c1-ee58-4cec-b3c9-e35f91954ff9: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-et-en-source+target + name: gpt-3-et-en-source+target reference: GPT-3 paper 75c32414-407b-4d6e-bcf1-d236457ffb90: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-et-en-target + name: how_to_say-et-en-target reference: '' 84b69786-4942-4805-8661-6bc2e710b3af: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-et-en-source+target + name: translate-et-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 8a5e785d-6e6d-498a-9a93-263653893c66: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-et-en-source+target + name: version-et-en-source+target reference: '' 8ea02fb0-bb15-43e6-8d0a-d4c0c79a6638: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_et-en-target + name: translate_as_et-en-target reference: '' 97073b9c-fffa-4ba4-9fff-0fb190735af0: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_et-en-source+target + name: translate_as_et-en-source+target reference: '' 9f14808e-8058-4949-8d74-44dacd83408e: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-et-source+starget + name: translate-en-et-source+starget reference: '' aba17c5d-94ee-4073-8ceb-72963fbfa9c2: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-et-target + name: gpt-3-en-et-target reference: GPT-3 paper b2cfff2d-27a1-4dcb-a544-82258b46825c: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-et-en-target + name: gpt-3-et-en-target reference: GPT-3 paper b3c91f4a-187b-45c7-ab92-16aa02a76547: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-et-source+target + name: version-en-et-source+target reference: '' b4178376-c25f-4de4-8855-02b182f9ed1b: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-et-target + name: how_to_say-en-et-target reference: '' bb6e89c9-5fd4-43f4-bdeb-ddf8b01b3531: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-et-en-target + name: translate-et-en-target reference: '' c14f01b7-69eb-44be-be41-4e5698c53de0: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-et-target + name: xglm-en-et-target reference: XGLM paper https://arxiv.org/abs/2112.10668 c5bda122-aefe-4ef0-88a8-c3c9d99aecbf: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-et-en-target + name: xglm-et-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 e736c3da-fb88-4969-b60c-e39a9f5d386d: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-et-source-target + name: xglm-en-et-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 e982d790-b547-412d-beeb-a2870606eb98: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-et-target + name: version-en-et-target reference: '' fe16099c-1c68-499f-b655-74fa4701d694: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-et-en-source+target + name: xglm-et-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt18/fi-en/templates.yaml b/promptsource/templates/wmt18/fi-en/templates.yaml index 81e90f38d..0229e53ef 100644 --- a/promptsource/templates/wmt18/fi-en/templates.yaml +++ b/promptsource/templates/wmt18/fi-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-target + name: translate-fi-en-target reference: '' 1b1d85d1-32b2-44a2-b3ec-2fea89d2c02e: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-target + name: xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 21ee5ae4-9c75-4298-83be-b038d5153501: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-target + name: a_good_translation-en-fi-target reference: '' 24edf4be-dbc0-460e-b125-8735ff8f35ca: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-source-target + name: xglm-en-fi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 3c191620-56a7-4df1-9ed1-0eca5d0e8616: !Template answer_choices: null @@ -67,8 +63,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-source+target + name: translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 4688e425-9c23-4ebf-a612-e45cf1c82c7e: !Template answer_choices: null @@ -81,8 +76,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-source+target + name: gpt-3-fi-en-source+target reference: GPT-3 paper 4cccb82d-2d62-4298-ae40-f61ed2beeb37: !Template answer_choices: null @@ -95,8 +89,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-source+target + name: xglm-fi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 51029e7c-4d7c-41bf-8679-a1c931eb5263: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-source+target + name: version-fi-en-source+target reference: '' 52bb4805-756f-4c74-8573-fe62001f708e: !Template answer_choices: null @@ -123,8 +115,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-source+target + name: a_good_translation-fi-en-source+target reference: '' 566b077a-9e88-480b-8c2c-962b07eb8dbd: !Template answer_choices: null @@ -137,8 +128,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-source+starget + name: translate-en-fi-source+starget reference: '' 58646943-00c3-4b1a-b539-abda1af73620: !Template answer_choices: null @@ -151,8 +141,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-target + name: gpt-3-en-fi-target reference: GPT-3 paper 6cbadfd5-e2d1-4ee4-8f78-dd028f1b4fc2: !Template answer_choices: null @@ -166,8 +155,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-source+target + name: a_good_translation-en-fi-source+target reference: '' 739bde08-a704-4204-9b72-6a4981f83991: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-target + name: gpt-3-fi-en-target reference: GPT-3 paper 8e8d032a-4f22-4cdf-8861-3a6fc79f148f: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-target + name: version-en-fi-target reference: '' 8f85f8aa-c698-4219-871a-35480c4c3311: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-source+target + name: translate_as_en-fi-source+target reference: '' 930addb7-0d80-4ddd-8f21-729693d8cb37: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-target + name: a_good_translation-fi-en-target reference: '' ac5207f1-7499-45d2-a570-9c6f4e751810: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-target + name: translate-en-fi-target reference: '' ae27d437-3c89-44d4-a268-586c5835d299: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-target + name: translate_as_fi-en-target reference: '' bcf156c8-8d81-46ea-b511-432537cbcefa: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-source+target + name: version-en-fi-source+target reference: '' c81d082d-c493-44db-b72a-d82616cbbe21: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-source+target + name: translate_as_fi-en-source+target reference: '' d8d03f61-9811-4af2-8ac2-c11356c30ac5: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-target + name: xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 df798200-71a8-4c46-8414-57b075a34825: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-target + name: version-fi-en-target reference: '' e44885b9-8c9c-4c5e-942d-851c9956cfae: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-source+target + name: gpt-3-en-fi-source+target reference: GPT 3 paper edc5f88a-155e-4650-96cc-c10c9ba7b7cf: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fi-target + name: how_to_say-en-fi-target reference: '' f657dc01-987f-4aa6-85a5-482f164750d9: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-target + name: translate_as_en-fi-target reference: '' fa40e79e-ba97-4a88-a39d-ee4e4f0aaba3: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fi-en-target + name: how_to_say-fi-en-target reference: '' diff --git a/promptsource/templates/wmt18/ru-en/templates.yaml b/promptsource/templates/wmt18/ru-en/templates.yaml index 3b43dc1bf..a1f721c00 100644 --- a/promptsource/templates/wmt18/ru-en/templates.yaml +++ b/promptsource/templates/wmt18/ru-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 0f0cb8d1-0967-4b39-98d9-90f7f83021c9: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 1128f6e3-4a33-4793-8ac8-c167a198149d: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' 1a45a068-7b2d-43c7-a998-63ee7f6f8e19: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' 2c67569d-bca3-4552-94fc-a963ab379d53: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 35bab45f-70b1-42b4-aaf5-f113228e60fb: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' 428e554a-22a5-437b-a5e8-946749978e27: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper 50ad6cb7-8b62-4144-bce0-8a7464f8f755: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' 6347c761-2ac5-42e2-8c82-6081417d1d79: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 65337a40-e02a-46ab-9c95-3bda83467260: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 6c3fad5f-bae7-473e-a8d8-0e5da0803bac: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' 6ead070d-cb04-4236-84f5-120f40c80e3a: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper 838a5379-088e-4d06-b065-d45d70c3d9b5: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' 8cd34877-7933-4b8c-922e-8e42f6efd9d8: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' 96d0086c-c7be-4513-8448-2ae104d8829e: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' 996c66b5-57db-4fdd-b5fb-c4517d3ac8f3: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' 9f9b91df-4a77-4e29-a82a-002ed8d83bf7: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' a985bdd7-e165-486a-880d-c68c32acb42c: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' ac5b39bf-8acc-42ed-bc0d-6723dcd8761e: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' b3ea9108-afd4-4ade-a80f-4f9c514910ec: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper b412f718-09ca-4003-9e6b-0abb2e1701c0: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 b4484ec0-f128-44c4-a639-49e26645f19f: !Template answer_choices: null @@ -311,8 +290,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' bd2c6a7d-09f2-4082-ba20-e53bc7fd7a89: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' c43b019f-3c4a-43ee-a3a1-6346d0beab8c: !Template answer_choices: null @@ -339,8 +316,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 c9a66968-e695-4b22-b83a-ddc89c6220de: !Template answer_choices: null @@ -353,8 +329,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' fba7ed4e-8cf6-4da3-b27a-43400a514ed8: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' diff --git a/promptsource/templates/wmt18/tr-en/templates.yaml b/promptsource/templates/wmt18/tr-en/templates.yaml index 81cc050ec..3dcd51ee1 100644 --- a/promptsource/templates/wmt18/tr-en/templates.yaml +++ b/promptsource/templates/wmt18/tr-en/templates.yaml @@ -11,8 +11,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-source+target + name: a_good_translation-tr-en-source+target reference: '' 251bf17b-2604-45d3-a875-8c08227de84a: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-target + name: translate_as_tr-en-target reference: '' 35036ac2-06b6-4899-89b2-c96436ab3c4e: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-target + name: xglm-tr-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 3588d4c9-4a11-41cb-b548-37d04f0f9b66: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-target + name: translate-en-tr-target reference: '' 3afad62b-2cab-4c06-965a-195588c5ef70: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-source+target + name: version-en-tr-source+target reference: '' 427c296e-ff3f-4d9c-8ed1-cc1900ad4d5f: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-target + name: translate_as_en-tr-target reference: '' 44842d8e-fb52-43cd-a042-2a5145e28b9a: !Template answer_choices: null @@ -95,8 +89,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-target + name: a_good_translation-en-tr-target reference: '' 5b01d692-c545-46d6-adce-5a889af5d71e: !Template answer_choices: null @@ -109,8 +102,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-target + name: gpt-3-en-tr-target reference: GPT-3 paper 5df2bcb3-45d7-47c1-ac4c-24ee23041c21: !Template answer_choices: null @@ -123,8 +115,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-tr-en-target + name: how_to_say-tr-en-target reference: '' 6a79649a-46aa-4048-9178-766117a9faa5: !Template answer_choices: null @@ -137,8 +128,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-source+target + name: translate-tr-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 700aa770-08cb-4fc5-b539-19cf26d8309b: !Template answer_choices: null @@ -151,8 +141,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-tr-target + name: how_to_say-en-tr-target reference: '' 71825d8e-79d6-4ca4-82c2-6647af66432c: !Template answer_choices: null @@ -166,8 +155,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-tr-target + name: version-en-tr-target reference: '' 7405c125-29ff-440a-bdc0-d1d8ffaff04b: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_tr-en-source+target + name: translate_as_tr-en-source+target reference: '' 76b7109c-6307-468c-8d14-23ac760fd22c: !Template answer_choices: null @@ -194,8 +181,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-tr-en-source+target + name: xglm-tr-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 85e15d3a-462a-4de9-a854-270a580d3909: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-tr-en-target + name: a_good_translation-tr-en-target reference: '' 9b08536a-517a-49b2-a446-636f0c2b608c: !Template answer_choices: null @@ -223,8 +208,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-source+target + name: gpt-3-tr-en-source+target reference: GPT-3 paper 9f577eb5-1d96-492b-843e-ea656dd56964: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-tr-en-target + name: gpt-3-tr-en-target reference: GPT-3 paper a7118387-ce27-4852-ae7a-634e07e493a7: !Template answer_choices: null @@ -251,8 +234,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-tr-en-target + name: translate-tr-en-target reference: '' ae1b2105-85a8-417e-8edf-87170aab6d27: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-target + name: version-tr-en-target reference: '' b6fe589c-bfea-433e-ab96-5b921693ebfa: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-tr-source+starget + name: translate-en-tr-source+starget reference: '' ca255baf-3552-40bf-b2c8-4bd3957b86a3: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-tr-source+target + name: gpt-3-en-tr-source+target reference: GPT 3 paper cdc4d883-eb4e-40bc-bd56-3043d3ca6303: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-target + name: xglm-en-tr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 dbac9603-dc7c-4d1f-827f-e5b70f3bd023: !Template answer_choices: null @@ -322,8 +300,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-tr-source+target + name: translate_as_en-tr-source+target reference: '' dbe3c495-165f-4d7d-af8f-97c2de1cba94: !Template answer_choices: null @@ -336,8 +313,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-tr-source-target + name: xglm-en-tr-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 e80e51a0-188c-4bfb-8d3e-00048b6ddaa6: !Template answer_choices: null @@ -351,8 +327,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-tr-en-source+target + name: version-tr-en-source+target reference: '' f7ac15b6-0d18-4d3c-b9f9-4ae893f8ae2c: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-tr-source+target + name: a_good_translation-en-tr-source+target reference: '' diff --git a/promptsource/templates/wmt18/zh-en/templates.yaml b/promptsource/templates/wmt18/zh-en/templates.yaml index 8a4de53c9..aeb327b5d 100644 --- a/promptsource/templates/wmt18/zh-en/templates.yaml +++ b/promptsource/templates/wmt18/zh-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-source-target + name: xglm-en-zh-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 3c40fb69-5e39-4e86-8041-7df857ca8eff: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-target + name: version-zh-en-target reference: '' 47b5e0fe-5d5c-4c2a-aa7d-19c6c057eaa6: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-target + name: translate_as_en-zh-target reference: '' 52a51e61-a646-4c87-9683-d37ba7b1e0ad: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-target + name: xglm-en-zh-target reference: XGLM paper https://arxiv.org/abs/2112.10668 53ab524b-2afa-46a0-adb8-48815fd93468: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-source+target + name: gpt-3-zh-en-source+target reference: GPT-3 paper 545a4ee8-9e7e-4b93-81a9-2a591b7bb0d4: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-target + name: gpt-3-en-zh-target reference: GPT-3 paper 575b7ead-1786-48fd-8759-6b9a773d42dc: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-zh-en-target + name: how_to_say-zh-en-target reference: '' 5f97369b-a08d-4ded-9ce0-bc86745a9b26: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-target + name: translate_as_zh-en-target reference: '' 6716518e-23f5-4dcd-9f57-108d909b67b7: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-target + name: translate-en-zh-target reference: '' 6c15965b-c093-4b8f-a798-d91c6bf84720: !Template answer_choices: null @@ -139,8 +130,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-target + name: translate-zh-en-target reference: '' 799657ec-3fa5-4764-a281-df90bf698178: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-source+target + name: translate-zh-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 804991ab-d1ca-4c11-867a-2bf502e97a6b: !Template answer_choices: null @@ -167,8 +156,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-source+target + name: gpt-3-en-zh-source+target reference: GPT 3 paper 84a1183a-6845-4160-84e1-29cdc9e3ca8d: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-target + name: a_good_translation-en-zh-target reference: '' 87ad8be4-cc19-439d-b215-63eaab5f0531: !Template answer_choices: null @@ -194,8 +181,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-source+target + name: translate_as_en-zh-source+target reference: '' 8a21adc7-a4a3-4f08-9682-4af13dd23ad1: !Template answer_choices: null @@ -208,8 +194,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-target + name: gpt-3-zh-en-target reference: GPT-3 paper 94aad9b2-a0c0-42f6-a9cb-593408dfc4da: !Template answer_choices: null @@ -222,8 +207,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-source+starget + name: translate-en-zh-source+starget reference: '' 981898d6-e80a-42fb-9b60-980a1cbf2bcd: !Template answer_choices: null @@ -237,8 +221,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-source+target + name: version-zh-en-source+target reference: '' a74116a0-1889-47a4-91a1-26fd5ebcf668: !Template answer_choices: null @@ -250,8 +233,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-source+target + name: a_good_translation-zh-en-source+target reference: '' c489e70f-d9d6-47f0-9ad5-02f0c21a65f9: !Template answer_choices: null @@ -264,8 +246,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-source+target + name: translate_as_zh-en-source+target reference: '' c7d75cc8-d7e9-4e12-99b1-308a7cb1c31e: !Template answer_choices: null @@ -279,8 +260,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-source+target + name: version-en-zh-source+target reference: '' cbc8501e-8bdd-4ddf-8b8f-dd9f05d7e152: !Template answer_choices: null @@ -294,8 +274,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-target + name: a_good_translation-zh-en-target reference: '' d08a7b31-7de6-41b9-ac2b-a34b4f7bc23c: !Template answer_choices: null @@ -308,8 +287,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-zh-target + name: how_to_say-en-zh-target reference: '' d7c43184-8622-4d39-88d8-de2097ac5817: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-source+target + name: a_good_translation-en-zh-source+target reference: '' dacf1578-eb48-42a2-9b4e-3352097b3220: !Template answer_choices: null @@ -337,8 +314,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-target + name: xglm-zh-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 f088cd1f-1bc0-4014-ac04-f9c77e1c1d2b: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-target + name: version-en-zh-target reference: '' fe9ed444-6445-4886-ae18-ff96932ae661: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-source+target + name: xglm-zh-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt19/de-en/templates.yaml b/promptsource/templates/wmt19/de-en/templates.yaml index 8e515fc13..e46e6b3ac 100644 --- a/promptsource/templates/wmt19/de-en/templates.yaml +++ b/promptsource/templates/wmt19/de-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-target + name: translate-en-de-target reference: '' 21cb3e62-e78e-460e-88fd-6895bf1e9faa: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-target + name: xglm-de-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 365f80d7-5e4f-40e5-8567-f4e60f2f71d9: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-source+target + name: translate_as_de-en-source+target reference: '' 446c8f26-354b-4be8-ad17-df74e46d5527: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-en-source+target + name: xglm-de-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 46e30fb6-b626-4846-85ce-8b66cec0eb70: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-source-target + name: xglm-en-de-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 51f607b9-09c1-430a-a05c-2c0c29e1f27f: !Template answer_choices: null @@ -81,8 +76,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-source+target + name: a_good_translation-de-en-source+target reference: '' 5650e2eb-b31d-4a66-90e5-3dc358b0a152: !Template answer_choices: null @@ -95,8 +89,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-target + name: gpt-3-de-en-target reference: GPT-3 paper 5cc18e97-9267-4e8c-9f89-354e595880a4: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-source+target + name: a_good_translation-en-de-source+target reference: '' 64a199ac-e338-4a03-9879-1f62c2698dcc: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-en-target + name: how_to_say-de-en-target reference: '' 650edf2a-2ddb-4c41-8868-c7b542acf4ba: !Template answer_choices: null @@ -137,8 +128,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-de-target + name: a_good_translation-en-de-target reference: '' 71386534-dabb-4fb4-88ba-86ae0062cab7: !Template answer_choices: null @@ -151,8 +141,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-target + name: translate-de-en-target reference: '' 71a31bbb-3cd8-4755-95b7-ac7f16d47e7b: !Template answer_choices: null @@ -165,8 +154,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-source+target + name: gpt-3-en-de-source+target reference: GPT 3 paper 78f7787e-47b5-4c9c-afd2-14252b9e006c: !Template answer_choices: null @@ -180,8 +168,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-target + name: version-en-de-target reference: '' 84e4c8cf-fea3-4b84-9b41-79ea64c1b014: !Template answer_choices: null @@ -195,8 +182,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-target + name: version-de-en-target reference: '' 95191307-4681-4973-aa29-3017ac7023fa: !Template answer_choices: null @@ -209,8 +195,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-de-source+starget + name: translate-en-de-source+starget reference: '' 953b1373-2207-4916-a583-9f69ba5af1ff: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-en-source+target + name: version-de-en-source+target reference: '' a0801f73-bba4-46da-a915-26a106d2c4bc: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-en-target + name: translate_as_de-en-target reference: '' a8c0831c-d260-4ef6-91e2-6cef49a80858: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-de-target + name: how_to_say-en-de-target reference: '' b324c249-11f6-47e6-8aff-da8f3066a083: !Template answer_choices: null @@ -266,8 +248,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-de-target + name: gpt-3-en-de-target reference: GPT-3 paper ba2af4e3-78a6-41b0-bd57-4f58196aa584: !Template answer_choices: null @@ -280,8 +261,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-de-target + name: xglm-en-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 d111aea3-c014-4114-9bf9-4135e33181c9: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-en-target + name: a_good_translation-de-en-target reference: '' d57cbdfd-3380-4318-8755-7f3e66381e1a: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-target + name: translate_as_en-de-target reference: '' e10830ef-4e58-4d17-b5e0-20961b960b95: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-de-source+target + name: translate_as_en-de-source+target reference: '' e3c62d5c-373d-45ca-a840-52ad4d6609d4: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-de-source+target + name: version-en-de-source+target reference: '' e3cacbbb-763f-43a4-9ca1-e05d0e914a14: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-en-source+target + name: gpt-3-de-en-source+target reference: GPT-3 paper e9816ae4-4c1c-4c7b-a0a4-5e63bbea6936: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-en-source+target + name: translate-de-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf diff --git a/promptsource/templates/wmt19/fi-en/templates.yaml b/promptsource/templates/wmt19/fi-en/templates.yaml index 18002b055..35a352c85 100644 --- a/promptsource/templates/wmt19/fi-en/templates.yaml +++ b/promptsource/templates/wmt19/fi-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-target + name: translate-fi-en-target reference: '' 05cd2f4d-c21d-47a0-b2c7-bc048b23d6e7: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-source+target + name: version-fi-en-source+target reference: '' 07140e32-85f0-4562-b61f-0b42480b4740: !Template answer_choices: null @@ -42,8 +40,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fi-en-target + name: version-fi-en-target reference: '' 0a8b9a3f-e108-48e3-ba56-26298c529e39: !Template answer_choices: null @@ -56,8 +53,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-target + name: translate-en-fi-target reference: '' 18d3b42e-dd08-4ff4-aaca-ae33aae6262a: !Template answer_choices: null @@ -71,8 +67,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-target + name: version-en-fi-target reference: '' 222806b8-6b75-4ebd-99aa-9b444448606d: !Template answer_choices: null @@ -85,8 +80,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-source-target + name: xglm-en-fi-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 228fedd6-f8e2-4b22-9d88-5dbd159121f0: !Template answer_choices: null @@ -99,8 +93,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-source+target + name: gpt-3-en-fi-source+target reference: GPT 3 paper 3795be9e-22af-4a75-864a-49203df483f5: !Template answer_choices: null @@ -113,8 +106,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-target + name: gpt-3-fi-en-target reference: GPT-3 paper 4d491547-9bc5-4197-a5da-8bd4fe36b5fb: !Template answer_choices: null @@ -127,8 +119,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-source+target + name: translate_as_fi-en-source+target reference: '' 50ae69bf-ee4c-4c08-80aa-0c237080a184: !Template answer_choices: null @@ -141,8 +132,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-source+target + name: xglm-fi-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 50b8f1a8-9b72-45b2-9e76-47a1cae9b05b: !Template answer_choices: null @@ -156,8 +146,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-fi-source+target + name: version-en-fi-source+target reference: '' 50e28f97-014f-4e94-a530-bcdc7b7689d6: !Template answer_choices: null @@ -171,8 +160,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-source+target + name: a_good_translation-en-fi-source+target reference: '' 5acb2abe-33c6-4304-853f-4e7f94143bc3: !Template answer_choices: null @@ -185,8 +173,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fi-en-target + name: how_to_say-fi-en-target reference: '' 5dc46f7a-bf1e-48ae-9098-20b9e4a20798: !Template answer_choices: null @@ -199,8 +186,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-fi-target + name: xglm-en-fi-target reference: XGLM paper https://arxiv.org/abs/2112.10668 6107fac1-bb73-405c-b60f-06bd20369b14: !Template answer_choices: null @@ -213,8 +199,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fi-en-source+target + name: gpt-3-fi-en-source+target reference: GPT-3 paper 679d6be7-8b8a-4dc4-b6bf-40f89898608b: !Template answer_choices: null @@ -226,8 +211,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-source+target + name: a_good_translation-fi-en-source+target reference: '' 7076f205-f4f8-4993-b28c-3f3a1e002479: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-fi-target + name: how_to_say-en-fi-target reference: '' 7afceb71-777a-4499-abc3-96af3a28447c: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-fi-source+starget + name: translate-en-fi-source+starget reference: '' 7db8e1b5-3d2b-4e49-b710-001e86451e2e: !Template answer_choices: null @@ -269,8 +251,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fi-en-target + name: a_good_translation-fi-en-target reference: '' 7f60153a-78fb-4658-ba0c-30f31c82e2eb: !Template answer_choices: null @@ -283,8 +264,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-source+target + name: translate_as_en-fi-source+target reference: '' a5e5fb83-1bc3-439e-b001-11269a3013e2: !Template answer_choices: null @@ -297,8 +277,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fi-en-target + name: xglm-fi-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 bdb3e1cf-723b-4dad-90a0-5e00591b48dd: !Template answer_choices: null @@ -311,8 +290,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-fi-target + name: gpt-3-en-fi-target reference: GPT-3 paper d0f69c91-295a-4e9a-b129-18b5338bf152: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-fi-target + name: a_good_translation-en-fi-target reference: '' d8523015-12ec-4fd3-a22f-3a745c07a0b3: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fi-en-source+target + name: translate-fi-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf e463a8f4-da3c-4355-aae3-b23048fc1afb: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-fi-target + name: translate_as_en-fi-target reference: '' ea43fbc9-3239-4ff2-82b9-9f2133d904fb: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fi-en-target + name: translate_as_fi-en-target reference: '' diff --git a/promptsource/templates/wmt19/fr-de/templates.yaml b/promptsource/templates/wmt19/fr-de/templates.yaml index bbe284cd8..03211a370 100644 --- a/promptsource/templates/wmt19/fr-de/templates.yaml +++ b/promptsource/templates/wmt19/fr-de/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-fr-target + name: xglm-de-fr-target reference: XGLM paper https://arxiv.org/abs/2112.10668 29a91408-85f4-4a55-ae0f-16338b5749a5: !Template answer_choices: null @@ -25,8 +24,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-fr-target + name: a_good_translation-de-fr-target reference: '' 2c9b4ab7-9c57-4525-969f-b951df2b2a94: !Template answer_choices: null @@ -39,8 +37,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-fr-target + name: gpt-3-de-fr-target reference: GPT-3 paper 2e3e5a7c-b484-49c2-9660-461da0264f5e: !Template answer_choices: null @@ -53,8 +50,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-fr-source+target + name: translate_as_de-fr-source+target reference: '' 3dfc0e89-8459-4498-b5ef-87fe9682d516: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fr-de-target + name: version-fr-de-target reference: '' 4101e307-ca38-4848-acd9-616d60e1b8dd: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-fr-source+target + name: version-de-fr-source+target reference: '' 44665ae6-6ca1-4e4e-9469-0ff1add72fd1: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-de-fr-source+target + name: a_good_translation-de-fr-source+target reference: '' 4718d5fd-333a-453a-ab9e-ade6c0eb23e9: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fr-de-source+target + name: xglm-fr-de-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 538dc918-e66b-444a-bb7b-62e73934bb02: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-fr-de-target + name: xglm-fr-de-target reference: XGLM paper https://arxiv.org/abs/2112.10668 576d28f5-4785-42f2-9431-b33026c43498: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fr-de-source+target + name: gpt-3-fr-de-source+target reference: GPT-3 paper 5ead1818-8c5d-4516-89b8-34b10c8371a7: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-fr-de-target + name: how_to_say-fr-de-target reference: '' 70631159-fef5-42fb-a154-019fb0d96cd4: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-fr-source+starget + name: translate-de-fr-source+starget reference: '' 72bd10ed-0fd9-4718-b175-c20c5bd9a70e: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fr-de-source+target + name: translate_as_fr-de-source+target reference: '' 758d7918-2931-4883-9b1c-aeb44227a6c4: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-de-fr-target + name: how_to_say-de-fr-target reference: '' 7974a5c0-0b58-4067-9c3f-4ef8be35a987: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fr-de-target + name: a_good_translation-fr-de-target reference: '' 80c1e81e-8e63-478d-bc72-adfce1f95016: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_de-fr-target + name: translate_as_de-fr-target reference: '' 844bacf0-2da2-4ffd-947d-8898145b6b09: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-de-fr-target + name: translate-de-fr-target reference: '' 85a1dd29-152e-4064-b97a-2af772ddeb55: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-fr-de-source+target + name: version-fr-de-source+target reference: '' 871db540-18be-4cc4-818d-9009ce8de896: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-de-fr-source-target + name: xglm-de-fr-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 92d8b55a-6267-4fa0-af26-f99b921444c5: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fr-de-source+target + name: translate-fr-de-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 96ab70c9-ad4a-4bd2-863b-8c8aaa27e881: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-fr-de-target + name: gpt-3-fr-de-target reference: GPT-3 paper a855a184-a86c-4824-920b-da6cf3fe8faf: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-fr-de-target + name: translate-fr-de-target reference: '' ade41a91-0578-4ce3-ba1a-3cb4537cd266: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-de-fr-target + name: version-de-fr-target reference: '' c9332f62-6328-4aec-805b-4fded0df7ffa: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-fr-de-source+target + name: a_good_translation-fr-de-source+target reference: '' d1e73e14-b677-4cec-b43f-497f4997747c: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_fr-de-target + name: translate_as_fr-de-target reference: '' d6b79b6b-be44-4494-be72-513904736115: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-de-fr-source+target + name: gpt-3-de-fr-source+target reference: GPT 3 paper diff --git a/promptsource/templates/wmt19/gu-en/templates.yaml b/promptsource/templates/wmt19/gu-en/templates.yaml index 65a42f9b0..b0ffb48aa 100644 --- a/promptsource/templates/wmt19/gu-en/templates.yaml +++ b/promptsource/templates/wmt19/gu-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-gu-en-target + name: version-gu-en-target reference: '' 11ba3712-b335-4c47-bcb6-c6cd19203981: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-gu-target + name: how_to_say-en-gu-target reference: '' 28443716-ea33-458c-8c9a-f055dd3a161a: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-gu-source+target + name: gpt-3-en-gu-source+target reference: GPT 3 paper 2e6c5141-6937-49f2-810e-4eb9bd9c4d3c: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-gu-source-target + name: xglm-en-gu-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 3881054b-1e24-4300-b219-4f870bd72de9: !Template answer_choices: null @@ -69,8 +65,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-gu-source+starget + name: translate-en-gu-source+starget reference: '' 504f3217-bec3-4015-8209-8ad7676ecb20: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-gu-target + name: version-en-gu-target reference: '' 55b2fd44-a3b0-404e-a687-f61d6401b155: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_gu-en-source+target + name: translate_as_gu-en-source+target reference: '' 5b3df10f-7504-43c6-a667-fc98b72a1dd2: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-gu-en-target + name: translate-gu-en-target reference: '' 5ff2be23-ef8b-489f-bb43-87a417afdf9b: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-gu-source+target + name: translate_as_en-gu-source+target reference: '' 6fcb810b-1f74-4d01-b065-a7a1ffd3a1ce: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-gu-en-target + name: xglm-gu-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 79ce60b6-a9aa-4499-bf1e-be4ba040739b: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-gu-target + name: translate-en-gu-target reference: '' 8597ba15-219a-4d23-aea7-8262b1548226: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-gu-en-source+target + name: version-gu-en-source+target reference: '' 9a033c1a-fefc-46bc-84b7-299a7724c985: !Template answer_choices: null @@ -184,8 +172,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-gu-source+target + name: a_good_translation-en-gu-source+target reference: '' 9dcf2e80-7b92-4117-a713-e2e299d9cf39: !Template answer_choices: null @@ -199,8 +186,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-gu-en-target + name: a_good_translation-gu-en-target reference: '' ae16add5-c066-40e7-a7c9-71926206d849: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-gu-en-source+target + name: a_good_translation-gu-en-source+target reference: '' b3f958e4-a4d9-4ba7-91e6-8958f5612a47: !Template answer_choices: null @@ -226,8 +211,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-gu-target + name: gpt-3-en-gu-target reference: GPT-3 paper c8441059-2e74-4248-862b-901e053d6c36: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-gu-en-target + name: how_to_say-gu-en-target reference: '' ccc7f258-d5ce-48d2-a833-9bfd4e42f42c: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-gu-en-source+target + name: gpt-3-gu-en-source+target reference: GPT-3 paper cf3bf7ca-02f5-44a1-ab32-ff453bc05c1a: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_gu-en-target + name: translate_as_gu-en-target reference: '' cf759928-85d0-4779-ae73-e3cc64c63a08: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-gu-target + name: a_good_translation-en-gu-target reference: '' d4921d87-f2c4-4f7b-98ac-f330a93b672b: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-gu-en-target + name: gpt-3-gu-en-target reference: GPT-3 paper e1337b9d-8f65-4ab7-bd13-e2a65cff164e: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-gu-en-source+target + name: xglm-gu-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 e85a53f5-a00d-4a2d-b6b3-3ae64230d6fd: !Template answer_choices: null @@ -323,8 +301,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-gu-target + name: xglm-en-gu-target reference: XGLM paper https://arxiv.org/abs/2112.10668 e860b831-fed6-439e-bafa-b9fc04fc8873: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-gu-source+target + name: version-en-gu-source+target reference: '' ec9cae8d-c7ee-4e41-bc44-faa289de9a74: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-gu-en-source+target + name: translate-gu-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf f5a969a6-5dca-47b1-b15a-6cb4d296cb1d: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-gu-target + name: translate_as_en-gu-target reference: '' diff --git a/promptsource/templates/wmt19/kk-en/templates.yaml b/promptsource/templates/wmt19/kk-en/templates.yaml index 91a05e502..dfad20431 100644 --- a/promptsource/templates/wmt19/kk-en/templates.yaml +++ b/promptsource/templates/wmt19/kk-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-kk-target + name: gpt-3-en-kk-target reference: GPT-3 paper 05b7fa2e-0a0e-4e7f-8fa7-71aa04f40dc6: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-kk-target + name: how_to_say-en-kk-target reference: '' 05d095c2-f150-46e3-a39b-1f06e01ca39e: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-kk-target + name: version-en-kk-target reference: '' 12139536-fb0e-43b5-a763-d9a6c84101ba: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-kk-en-source+target + name: xglm-kk-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 29f05e00-9b0f-4db0-9226-f558fd02709d: !Template answer_choices: null @@ -70,8 +66,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-kk-en-source+target + name: version-kk-en-source+target reference: '' 2f05a859-a361-4e6a-ac89-48c2b66f0382: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-kk-source+starget + name: translate-en-kk-source+starget reference: '' 35258de4-8943-4a5f-b45c-72559fc048bc: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-kk-target + name: translate_as_en-kk-target reference: '' 361734f8-dc83-4194-87ad-937a210cc037: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-kk-en-target + name: gpt-3-kk-en-target reference: GPT-3 paper 3991795b-1860-4ffa-9001-26454c7174b2: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-kk-source-target + name: xglm-en-kk-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 45807ca5-c5df-4a43-a7f1-3935d49dbbe7: !Template answer_choices: null @@ -141,8 +132,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-kk-en-target + name: version-kk-en-target reference: '' 48ac727e-ecd6-4458-a8e2-9c1e1e9d1f73: !Template answer_choices: null @@ -156,8 +146,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-kk-source+target + name: version-en-kk-source+target reference: '' 5abf7d89-6eca-4d84-809a-1897af755a42: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-kk-en-source+target + name: a_good_translation-kk-en-source+target reference: '' 6b67073b-86ff-41f5-ace3-87380619aa1a: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-kk-target + name: translate-en-kk-target reference: '' 7a00cb3d-d7e9-40e0-b2f1-a639caef4105: !Template answer_choices: null @@ -197,8 +184,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_kk-en-target + name: translate_as_kk-en-target reference: '' 7e23e55c-1e67-4a21-98e3-a361eb989e98: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-kk-en-target + name: xglm-kk-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 92d739d1-ad1c-4379-bd0f-6d2584bb6147: !Template answer_choices: null @@ -224,8 +209,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-kk-target + name: a_good_translation-en-kk-target reference: '' a2528e1c-b6e5-4c2d-95ce-395d6d5dccc7: !Template answer_choices: null @@ -238,8 +222,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_kk-en-source+target + name: translate_as_kk-en-source+target reference: '' ac7eb2a0-7979-4045-bd86-65501805c2e0: !Template answer_choices: null @@ -252,8 +235,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-kk-en-source+target + name: translate-kk-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf bdac0d10-e077-4c76-9f7e-f6b6878e41ad: !Template answer_choices: null @@ -267,8 +249,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-kk-source+target + name: a_good_translation-en-kk-source+target reference: '' c03acc66-b187-4746-b5fe-4b14ab4ef417: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-kk-source+target + name: translate_as_en-kk-source+target reference: '' c7bc8a8c-829e-4863-a978-0ac6aee572ff: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-kk-en-target + name: translate-kk-en-target reference: '' d263f205-a7e9-450f-a821-6cb5dd167585: !Template answer_choices: null @@ -309,8 +288,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-kk-target + name: xglm-en-kk-target reference: XGLM paper https://arxiv.org/abs/2112.10668 e59ffaa7-d828-463f-b79d-6fe0e0378daa: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-kk-en-target + name: a_good_translation-kk-en-target reference: '' e614b3f0-ac5f-4570-ab95-8f246be0f077: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-kk-en-target + name: how_to_say-kk-en-target reference: '' f617d102-ce28-4b31-aae9-5ffd6341a8ba: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-kk-source+target + name: gpt-3-en-kk-source+target reference: GPT 3 paper ff749f28-c9d2-4830-a8bf-c7895e00aaaa: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-kk-en-source+target + name: gpt-3-kk-en-source+target reference: GPT-3 paper diff --git a/promptsource/templates/wmt19/lt-en/templates.yaml b/promptsource/templates/wmt19/lt-en/templates.yaml index c127ea6f0..9f095ecef 100644 --- a/promptsource/templates/wmt19/lt-en/templates.yaml +++ b/promptsource/templates/wmt19/lt-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-lt-en-source+target + name: translate-lt-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 0380efec-57ed-4437-84dc-a61f79017856: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_lt-en-source+target + name: translate_as_lt-en-source+target reference: '' 0ac8a6cc-615a-4b37-864f-66a223c5dbd9: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-lt-en-source+target + name: xglm-lt-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 18c81ecf-7582-49a8-a4c8-5bb4281ce98f: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-lt-source+target + name: gpt-3-en-lt-source+target reference: GPT 3 paper 1a6f9056-57b3-41ab-a365-a7932bee428f: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-lt-en-target + name: how_to_say-lt-en-target reference: '' 1ce4b380-5230-41f7-b8bb-824cacdf80ee: !Template answer_choices: null @@ -83,8 +78,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-lt-target + name: version-en-lt-target reference: '' 264b2535-2e7d-4efe-9704-09c2b1332cac: !Template answer_choices: null @@ -97,8 +91,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-lt-target + name: translate-en-lt-target reference: '' 28987179-bc04-4e63-982a-b959efbd95a5: !Template answer_choices: null @@ -110,8 +103,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-lt-target + name: a_good_translation-en-lt-target reference: '' 4bb2c423-342a-46fc-928b-eb4d9a01297c: !Template answer_choices: null @@ -124,8 +116,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-lt-en-target + name: xglm-lt-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 4f5f0594-32a5-4690-81ae-64deb3289f94: !Template answer_choices: null @@ -138,8 +129,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-lt-en-target + name: translate-lt-en-target reference: '' 5afbe8cf-9f27-4482-b936-44a06fae2a21: !Template answer_choices: null @@ -153,8 +143,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-lt-en-source+target + name: version-lt-en-source+target reference: '' 615c0d2b-ead7-458a-b3b8-16092d899d0f: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-lt-source+target + name: version-en-lt-source+target reference: '' 74fb4a3d-99a1-4bd0-a57d-2bd53732eb35: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-lt-source-target + name: xglm-en-lt-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 77a3ab9f-f3ce-4b46-b080-13776b973919: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-lt-target + name: how_to_say-en-lt-target reference: '' 8d69a032-f7c1-49ff-acea-093b672c9dd6: !Template answer_choices: null @@ -210,8 +196,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-lt-target + name: gpt-3-en-lt-target reference: GPT-3 paper 97a087e8-fd66-4b95-9c75-87250b8e5e5c: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-lt-en-target + name: a_good_translation-lt-en-target reference: '' a09584a3-544a-4508-9890-7db646ec6902: !Template answer_choices: null @@ -240,8 +224,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-lt-source+target + name: a_good_translation-en-lt-source+target reference: '' b4b2863d-0eb4-4220-86ba-cd7873a31e33: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-lt-source+target + name: translate_as_en-lt-source+target reference: '' ba8c242b-aeac-44b0-8ed0-4e108fa46ddc: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-lt-en-source+target + name: gpt-3-lt-en-source+target reference: GPT-3 paper bb388f4f-0184-4ee1-b445-6ce35a4566a4: !Template answer_choices: null @@ -282,8 +263,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-lt-en-target + name: gpt-3-lt-en-target reference: GPT-3 paper c2fb9339-44a0-468b-aa07-984aaf09cb9c: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-lt-en-source+target + name: a_good_translation-lt-en-source+target reference: '' c8412be4-f223-47d1-ad3f-36c640edaaaa: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-lt-en-target + name: version-lt-en-target reference: '' cab29758-8001-4d7f-9eaa-b492cb0aca49: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_lt-en-target + name: translate_as_lt-en-target reference: '' da751307-9d0b-4566-bb95-9fa90f0132f5: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-lt-target + name: translate_as_en-lt-target reference: '' e5b2d433-eff7-4ca6-b10c-5cc3b3c5d662: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-lt-target + name: xglm-en-lt-target reference: XGLM paper https://arxiv.org/abs/2112.10668 f3fbfb49-72a6-4aa6-b48e-f3adf02e54d5: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-lt-source+starget + name: translate-en-lt-source+starget reference: '' diff --git a/promptsource/templates/wmt19/ru-en/templates.yaml b/promptsource/templates/wmt19/ru-en/templates.yaml index 9c1247d7a..2edb93dc6 100644 --- a/promptsource/templates/wmt19/ru-en/templates.yaml +++ b/promptsource/templates/wmt19/ru-en/templates.yaml @@ -13,8 +13,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-source+target + name: version-en-ru-source+target reference: '' 1975db74-601b-40e5-8615-f1af51e11857: !Template answer_choices: null @@ -27,8 +26,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-ru-target + name: how_to_say-en-ru-target reference: '' 1b9be0af-e67d-4f87-b032-172faa9e329b: !Template answer_choices: null @@ -41,8 +39,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-target + name: gpt-3-ru-en-target reference: GPT-3 paper 2124a999-8d8e-4ba5-8eab-5e475aff39cc: !Template answer_choices: null @@ -55,8 +52,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-target + name: translate-en-ru-target reference: '' 256e0bdd-c8f8-4561-b6f6-e72c7acaa9c6: !Template answer_choices: null @@ -70,8 +66,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-source+target + name: a_good_translation-en-ru-source+target reference: '' 2c9a43f2-6aa1-438d-b6eb-6863ff34cc76: !Template answer_choices: null @@ -84,8 +79,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-target + name: translate_as_en-ru-target reference: '' 314b2f22-b592-4adb-8a41-2e88c5ab9a3e: !Template answer_choices: null @@ -98,8 +92,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-target + name: xglm-en-ru-target reference: XGLM paper https://arxiv.org/abs/2112.10668 32681ba6-ad58-403c-b758-3d3793477327: !Template answer_choices: null @@ -112,8 +105,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-target + name: gpt-3-en-ru-target reference: GPT-3 paper 3cc7638d-91cc-4838-a4a4-beb612ee2051: !Template answer_choices: null @@ -126,8 +118,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-source+target + name: translate-ru-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 3e7069c6-6efb-4865-b18e-e67bd4425c5c: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-ru-source+target + name: translate_as_en-ru-source+target reference: '' 46b88a9c-4e5e-4479-9840-85454dbcb303: !Template answer_choices: null @@ -154,8 +144,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-target + name: translate_as_ru-en-target reference: '' 4861eaf1-a385-4b2c-97e7-47fd0f484973: !Template answer_choices: null @@ -168,8 +157,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-ru-en-target + name: translate-ru-en-target reference: '' 56a6ef90-d916-4872-9971-28d32142af96: !Template answer_choices: null @@ -182,8 +170,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-ru-en-source+target + name: gpt-3-ru-en-source+target reference: GPT-3 paper 5e34c936-eb95-49fd-abf8-76e069250e6a: !Template answer_choices: null @@ -196,8 +183,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-ru-source+target + name: gpt-3-en-ru-source+target reference: GPT 3 paper 62516534-2a39-47fb-8403-5c6e7dd1c97b: !Template answer_choices: null @@ -211,8 +197,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-source+target + name: version-ru-en-source+target reference: '' 6459ffc2-6c60-459e-82db-84199a66362d: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-source+target + name: xglm-ru-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668 7aa83e54-ffb0-4c9e-8aea-4d91237897db: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-ru-en-target + name: xglm-ru-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 81b4a43f-b096-4b0e-944d-9a23f0110a2b: !Template answer_choices: null @@ -253,8 +236,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-ru-en-target + name: how_to_say-ru-en-target reference: '' 89e8ef4e-f0e9-4e61-970a-ab6e6e515387: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-target + name: a_good_translation-ru-en-target reference: '' 9488c5d3-20bb-4044-a1de-ca0bda231ad2: !Template answer_choices: null @@ -283,8 +264,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-ru-target + name: version-en-ru-target reference: '' aa3c33be-469b-4a65-9aa9-3d67552b8455: !Template answer_choices: null @@ -296,8 +276,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-ru-target + name: a_good_translation-en-ru-target reference: '' ac0c8454-6ec4-4743-b31d-768279cbf9e3: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-ru-source+starget + name: translate-en-ru-source+starget reference: '' af6bced2-3292-4e85-8434-44bf0cebc14b: !Template answer_choices: null @@ -325,8 +303,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-ru-en-target + name: version-ru-en-target reference: '' afa124a1-d7b0-4b5b-8d5d-8d20f47aa6aa: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-ru-en-source+target + name: a_good_translation-ru-en-source+target reference: '' bf638811-ea72-40e7-a9d2-68fe34e41422: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_ru-en-source+target + name: translate_as_ru-en-source+target reference: '' cc4e99c5-9172-486c-960c-5708e960210b: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-ru-source-target + name: xglm-en-ru-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 diff --git a/promptsource/templates/wmt19/zh-en/templates.yaml b/promptsource/templates/wmt19/zh-en/templates.yaml index 002097f7e..e13ce2001 100644 --- a/promptsource/templates/wmt19/zh-en/templates.yaml +++ b/promptsource/templates/wmt19/zh-en/templates.yaml @@ -12,8 +12,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-source+starget + name: translate-en-zh-source+starget reference: '' 03bf5bf1-50dc-4278-ae77-c21fc9bb5b38: !Template answer_choices: null @@ -26,8 +25,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-target + name: gpt-3-en-zh-target reference: GPT-3 paper 08d528bb-62b8-407f-a917-cd8f2f039dc0: !Template answer_choices: null @@ -40,8 +38,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-target + name: gpt-3-zh-en-target reference: GPT-3 paper 0c961ed2-1fd4-425a-87cf-4f597464afd3: !Template answer_choices: null @@ -54,8 +51,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-en-zh-source+target + name: gpt-3-en-zh-source+target reference: GPT 3 paper 13411150-33f4-48db-b6c3-e7704956d6c8: !Template answer_choices: null @@ -68,8 +64,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-source+target + name: translate_as_en-zh-source+target reference: '' 1e87b260-0ac0-45dd-aff6-dc1ee1403f6b: !Template answer_choices: null @@ -82,8 +77,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-en-zh-target + name: translate-en-zh-target reference: '' 2a63c023-4fb7-41d2-bb84-5fad3bf7f6ac: !Template answer_choices: null @@ -96,8 +90,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-zh-en-target + name: how_to_say-zh-en-target reference: '' 3204009f-5f11-449b-b31c-76cf4b099025: !Template answer_choices: null @@ -111,8 +104,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-source+target + name: version-zh-en-source+target reference: '' 328efda5-1fc0-4a75-96c2-351a8669ca6b: !Template answer_choices: null @@ -125,8 +117,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-target + name: translate_as_zh-en-target reference: '' 4487fae5-dddd-482a-bdd7-91a37a7537dd: !Template answer_choices: null @@ -140,8 +131,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-source+target + name: version-en-zh-source+target reference: '' 5269bb03-015a-4db0-a5fa-9337d95c81e5: !Template answer_choices: null @@ -155,8 +145,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-source+target + name: a_good_translation-en-zh-source+target reference: '' 6b066007-3a6b-415f-b8ef-bb93f894f831: !Template answer_choices: null @@ -169,8 +158,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-source-target + name: xglm-en-zh-source-target reference: Adapted from XGLM paper on few shot evaluation https://arxiv.org/abs/2112.10668 72c78300-19a2-4aad-83e9-e5b30a46dafd: !Template answer_choices: null @@ -183,8 +171,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-en-zh-target + name: xglm-en-zh-target reference: XGLM paper https://arxiv.org/abs/2112.10668 743eb883-26f3-4868-b8c0-a7d5b83776c6: !Template answer_choices: null @@ -198,8 +185,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-zh-en-target + name: version-zh-en-target reference: '' 7f599e74-5e22-452d-a661-111f3388e136: !Template answer_choices: null @@ -212,8 +198,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - gpt-3-zh-en-source+target + name: gpt-3-zh-en-source+target reference: GPT-3 paper 8040aec8-e681-47f3-b204-c18ec4f78f5e: !Template answer_choices: null @@ -225,8 +210,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-en-zh-target + name: a_good_translation-en-zh-target reference: '' 868c0495-d871-418d-8ff3-a8d34309721f: !Template answer_choices: null @@ -239,8 +223,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-source+target + name: translate-zh-en-source+target reference: https://arxiv.org/pdf/1910.10683.pdf 88bd8591-5b26-4f2b-9a12-8e57995bbaab: !Template answer_choices: null @@ -254,8 +237,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - version-en-zh-target + name: version-en-zh-target reference: '' 89ee6d56-84c2-4e44-8b70-2da7a8054f9b: !Template answer_choices: null @@ -268,8 +250,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - how_to_say-en-zh-target + name: how_to_say-en-zh-target reference: '' 98dde747-820b-4ec9-b9ab-fc76aba0c867: !Template answer_choices: null @@ -281,8 +262,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-source+target + name: a_good_translation-zh-en-source+target reference: '' ac3760e1-d074-43d4-8199-b19a1ee3effb: !Template answer_choices: null @@ -295,8 +275,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_zh-en-source+target + name: translate_as_zh-en-source+target reference: '' bccb49ca-d1b7-4930-b02d-52d8cbfbe6a5: !Template answer_choices: null @@ -310,8 +289,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - a_good_translation-zh-en-target + name: a_good_translation-zh-en-target reference: '' bd22ec10-1e72-45d0-93b0-edc16a567097: !Template answer_choices: null @@ -324,8 +302,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate_as_en-zh-target + name: translate_as_en-zh-target reference: '' e1275325-ec0c-407f-a13d-903260f66347: !Template answer_choices: null @@ -338,8 +315,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - translate-zh-en-target + name: translate-zh-en-target reference: '' e196c412-aeea-4a1d-bdcb-992734772e07: !Template answer_choices: null @@ -352,8 +328,7 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-target + name: xglm-zh-en-target reference: XGLM paper https://arxiv.org/abs/2112.10668 e4aaa404-3e92-4469-a074-b671442e049f: !Template answer_choices: null @@ -366,6 +341,5 @@ templates: metrics: - BLEU original_task: true - name: !!python/tuple - - xglm-zh-en-source+target + name: xglm-zh-en-source+target reference: XGLM paper https://arxiv.org/abs/2112.10668