From 8206ffc610f3971f1ba47e38d518cdf1e10ac582 Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 15:35:34 -0400 Subject: [PATCH 01/17] add EN prompts for xcopa --- .../templates/xcopa/id/templates.yaml | 211 ++++++++++++++++++ .../templates/xcopa/it/templates.yaml | 211 ++++++++++++++++++ .../templates/xcopa/sw/templates.yaml | 211 ++++++++++++++++++ .../templates/xcopa/ta/templates.yaml | 211 ++++++++++++++++++ .../templates/xcopa/vi/templates.yaml | 211 ++++++++++++++++++ .../templates/xcopa/zh/templates.yaml | 211 ++++++++++++++++++ 6 files changed, 1266 insertions(+) create mode 100644 promptsource/templates/xcopa/id/templates.yaml create mode 100644 promptsource/templates/xcopa/it/templates.yaml create mode 100644 promptsource/templates/xcopa/sw/templates.yaml create mode 100644 promptsource/templates/xcopa/ta/templates.yaml create mode 100644 promptsource/templates/xcopa/vi/templates.yaml create mode 100644 promptsource/templates/xcopa/zh/templates.yaml diff --git a/promptsource/templates/xcopa/id/templates.yaml b/promptsource/templates/xcopa/id/templates.yaml new file mode 100644 index 000000000..db6086413 --- /dev/null +++ b/promptsource/templates/xcopa/id/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: id +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Indonesian sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/promptsource/templates/xcopa/it/templates.yaml b/promptsource/templates/xcopa/it/templates.yaml new file mode 100644 index 000000000..00065a55e --- /dev/null +++ b/promptsource/templates/xcopa/it/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: it +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Italian sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/promptsource/templates/xcopa/sw/templates.yaml b/promptsource/templates/xcopa/sw/templates.yaml new file mode 100644 index 000000000..f7a161418 --- /dev/null +++ b/promptsource/templates/xcopa/sw/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: sw +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Swahili sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/promptsource/templates/xcopa/ta/templates.yaml b/promptsource/templates/xcopa/ta/templates.yaml new file mode 100644 index 000000000..5d5d95b98 --- /dev/null +++ b/promptsource/templates/xcopa/ta/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: ta +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Tamil sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/promptsource/templates/xcopa/vi/templates.yaml b/promptsource/templates/xcopa/vi/templates.yaml new file mode 100644 index 000000000..1d78885dc --- /dev/null +++ b/promptsource/templates/xcopa/vi/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: vi +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Vietnamese sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' diff --git a/promptsource/templates/xcopa/zh/templates.yaml b/promptsource/templates/xcopa/zh/templates.yaml new file mode 100644 index 000000000..4d5792f78 --- /dev/null +++ b/promptsource/templates/xcopa/zh/templates.yaml @@ -0,0 +1,211 @@ +dataset: xcopa +subset: zh +templates: + 0edd8660-f299-4819-a5ac-633c11177228: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 0edd8660-f299-4819-a5ac-633c11177228 + jinja: 'Exercise: Choose the most plausible completion to this Chinese sentence: + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: exercise + reference: '' + 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026What could happen next, C1 or C2?" + reference: '' + 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: i_am_hesitating + reference: '' + 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: plausible_alternatives + reference: '' + 744047dc-1298-45a2-8d68-d67e3f834ded: !Template + answer_choices: '{{choice1 }} ||| {{choice2}}' + id: 744047dc-1298-45a2-8d68-d67e3f834ded + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% + if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != + -1 %}{{ answer_choices[label] }}{% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026As a result, C1 or C2?" + reference: '' + 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: best_option + reference: '' + 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026which may be caused by" + reference: '' + a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + jinja: 'Pick the more likely continuation to the following sentence: + + {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: more likely + reference: '' + a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a61d8c21-da25-47bf-b5fe-14a8edd650af + jinja: '{{ premise }} + + + Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + {% endif %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: cause_effect + reference: '' + a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: "\u2026why? C1 or C2" + reference: '' + f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: f32348cd-d3cb-4619-87b9-e24f99c78567 + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} + + Choose between: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: choose + reference: '' From 6af75cb7114224be459bcedf62f7b5f10e5881a0 Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 16:57:31 -0400 Subject: [PATCH 02/17] update ids (2/5) --- .../templates/xcopa/id/templates.yaml | 166 +++++++++--------- .../templates/xcopa/it/templates.yaml | 66 +++---- 2 files changed, 116 insertions(+), 116 deletions(-) diff --git a/promptsource/templates/xcopa/id/templates.yaml b/promptsource/templates/xcopa/id/templates.yaml index db6086413..8f78da924 100644 --- a/promptsource/templates/xcopa/id/templates.yaml +++ b/promptsource/templates/xcopa/id/templates.yaml @@ -1,18 +1,13 @@ dataset: xcopa subset: id templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 1a87b487-1570-4873-aed9-b84d2fc0476c: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Indonesian sentence: - - - {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif - %} - - - {{choice1}} - - - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + id: 1a87b487-1570-4873-aed9-b84d2fc0476c + jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ + \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ + \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -20,14 +15,14 @@ templates: metrics: - Accuracy original_task: true - name: exercise + name: i_am_hesitating reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 336c4c72-40e3-4122-881e-8cd7a1881eec: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b - jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ - \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ - \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + id: 336c4c72-40e3-4122-881e-8cd7a1881eec + jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ + \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ + \ }}{%endif%}\n{% endif %}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -35,15 +30,14 @@ templates: metrics: - Accuracy original_task: true - name: "\u2026What could happen next, C1 or C2?" + name: "\u2026why? C1 or C2" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 482f0b87-e748-4e98-8cc8-a23386bc50c3: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba - jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ - \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ - \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ - \ }}{%endif%}" + id: 482f0b87-e748-4e98-8cc8-a23386bc50c3 + jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ + \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -51,15 +45,15 @@ templates: metrics: - Accuracy original_task: true - name: i_am_hesitating + name: best_option reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 4a0640a5-c378-422d-879b-7490bc500c8a: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d - jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% - else %} As a consequence... {% endif %} + id: 4a0640a5-c378-422d-879b-7490bc500c8a + jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... + {% endif %} - Help me pick the more plausible option: + Choose between: - {{choice1}} @@ -71,14 +65,19 @@ templates: metrics: - Accuracy original_task: true - name: plausible_alternatives + name: choose reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 78e28a66-a84c-442c-9bf7-44aa49450412: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 78e28a66-a84c-442c-9bf7-44aa49450412 + jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% + else %} As a consequence... {% endif %} + + Help me pick the more plausible option: + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -86,11 +85,11 @@ templates: metrics: - Accuracy original_task: true - name: "C1 or C2? premise, so/because\u2026" - reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + name: plausible_alternatives + reference: '' + 7c0b578c-214f-4dc9-a9b4-252d91691cb0: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 7c0b578c-214f-4dc9-a9b4-252d91691cb0 jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,12 +102,18 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + 94b5be71-c989-4a62-96d9-a7cb042e83c7: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 - jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ - \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" + id: 94b5be71-c989-4a62-96d9-a7cb042e83c7 + jinja: 'Exercise: choose the most plausible alternative. + + + {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif + %} + + - {{choice1}} + + - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -116,14 +121,14 @@ templates: metrics: - Accuracy original_task: true - name: best_option + name: exercise reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + b308f6ce-673c-44c1-b84d-95a3045229ea: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb - jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ - \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ - \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" + id: b308f6ce-673c-44c1-b84d-95a3045229ea + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -131,14 +136,15 @@ templates: metrics: - Accuracy original_task: true - name: "\u2026which may be caused by" - reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + name: "C1 or C2? premise, so/because\u2026" + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + cf78cf75-90cc-4fe2-8b78-2bf64c9520b4: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 - jinja: 'Pick the more likely continuation to the following sentence: + id: cf78cf75-90cc-4fe2-8b78-2bf64c9520b4 + jinja: '{{ premise }} - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -151,20 +157,14 @@ templates: metrics: - Accuracy original_task: true - name: more likely + name: cause_effect reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + d8263afb-215f-43c4-83b8-c85744144fdb: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af - jinja: '{{ premise }} - - - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: - {% endif %} - - - {{choice1}} - - - {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label] }}{%endif%}' + id: d8263afb-215f-43c4-83b8-c85744144fdb + jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -172,14 +172,14 @@ templates: metrics: - Accuracy original_task: true - name: cause_effect + name: "\u2026which may be caused by" reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + eaddf2e0-ead4-456b-8e81-00bdcde8c7b0: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e - jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ - \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ - \ }}{%endif%}\n{% endif %}" + id: eaddf2e0-ead4-456b-8e81-00bdcde8c7b0 + jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ + \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ + \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -187,15 +187,15 @@ templates: metrics: - Accuracy original_task: true - name: "\u2026why? C1 or C2" + name: "\u2026What could happen next, C1 or C2?" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + ebd4242a-14f2-4aed-a183-dc37a18dfe4b: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 - jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... - {% endif %} + id: ebd4242a-14f2-4aed-a183-dc37a18dfe4b + jinja: 'Pick the more likely continuation to the following sentence: - Choose between: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: + {% endif %} - {{choice1}} @@ -207,5 +207,5 @@ templates: metrics: - Accuracy original_task: true - name: choose + name: more likely reference: '' diff --git a/promptsource/templates/xcopa/it/templates.yaml b/promptsource/templates/xcopa/it/templates.yaml index 00065a55e..99141407d 100644 --- a/promptsource/templates/xcopa/it/templates.yaml +++ b/promptsource/templates/xcopa/it/templates.yaml @@ -1,10 +1,10 @@ dataset: xcopa subset: it templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 1d23192b-7276-4e6b-9e21-c65953f42f4d: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Italian sentence: + id: 1d23192b-7276-4e6b-9e21-c65953f42f4d + jinja: 'Exercise: choose the most plausible alternative. {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif @@ -22,9 +22,9 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 2a2a351a-bae3-426c-9f3d-8908a200159e: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + id: 2a2a351a-bae3-426c-9f3d-8908a200159e jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -37,9 +37,9 @@ templates: original_task: true name: "\u2026What could happen next, C1 or C2?" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 4866dc0c-b91c-44a9-8436-6e227e545859: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + id: 4866dc0c-b91c-44a9-8436-6e227e545859 jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ @@ -53,9 +53,9 @@ templates: original_task: true name: i_am_hesitating reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 49e8635c-4e5a-48e5-a52a-12aa5b5d8d27: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + id: 49e8635c-4e5a-48e5-a52a-12aa5b5d8d27 jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% else %} As a consequence... {% endif %} @@ -73,12 +73,12 @@ templates: original_task: true name: plausible_alternatives reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 5781b266-6eb9-49aa-880c-9669cbdb4017: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 5781b266-6eb9-49aa-880c-9669cbdb4017 + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -88,9 +88,9 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + 85ffa482-d662-497f-bdcd-586b60918d37: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 85ffa482-d662-497f-bdcd-586b60918d37 jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,11 +103,11 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + 9b9cc9e5-2ca9-43f5-9a1f-b5831ed5d77d: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + id: 9b9cc9e5-2ca9-43f5-9a1f-b5831ed5d77d jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true @@ -118,9 +118,9 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + b5206dd3-7856-41fc-9671-5a22de303334: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + id: b5206dd3-7856-41fc-9671-5a22de303334 jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -133,12 +133,12 @@ templates: original_task: true name: "\u2026which may be caused by" reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template - answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + b90379c6-98d6-436e-a0db-466ad0f5a1c1: !Template + answer_choices: '{{choice1}} ||| {{choice2}}'s + id: b90379c6-98d6-436e-a0db-466ad0f5a1c1 jinja: 'Pick the more likely continuation to the following sentence: - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: {% endif %} - {{choice1}} @@ -153,13 +153,13 @@ templates: original_task: true name: more likely reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + bd78db08-6b60-40e4-840b-b2495153c738: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af + id: bd78db08-6b60-40e4-840b-b2495153c738 jinja: '{{ premise }} - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -174,9 +174,9 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + d016e3c4-45ae-4711-a360-71ab8e139e91: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + id: d016e3c4-45ae-4711-a360-71ab8e139e91 jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -189,9 +189,9 @@ templates: original_task: true name: "\u2026why? C1 or C2" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + f5dc9303-918b-485e-927c-76c8fc76c55f: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 + id: f5dc9303-918b-485e-927c-76c8fc76c55f jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif %} From 1d4d0cc2cb4440b61361e530f0496cc50ee3f5d7 Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 17:05:27 -0400 Subject: [PATCH 03/17] regenerate ids (4/6) --- .../templates/xcopa/sw/templates.yaml | 66 +++++++++---------- .../templates/xcopa/ta/templates.yaml | 63 +++++++++--------- 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/promptsource/templates/xcopa/sw/templates.yaml b/promptsource/templates/xcopa/sw/templates.yaml index f7a161418..98b85f66c 100644 --- a/promptsource/templates/xcopa/sw/templates.yaml +++ b/promptsource/templates/xcopa/sw/templates.yaml @@ -1,10 +1,10 @@ dataset: xcopa subset: sw templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 18089f74-c42c-4d6d-8eb0-9dad49b9f704: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Swahili sentence: + id: 18089f74-c42c-4d6d-8eb0-9dad49b9f704 + jinja: 'Exercise: choose the most plausible alternative. {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif @@ -22,9 +22,9 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 2464779b-7773-4d4f-99b9-03229740de4a: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + id: 2464779b-7773-4d4f-99b9-03229740de4a jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -37,9 +37,9 @@ templates: original_task: true name: "\u2026What could happen next, C1 or C2?" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 38729cda-f248-4788-82e5-71331256ca73: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + id: 38729cda-f248-4788-82e5-71331256ca73 jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ @@ -53,9 +53,9 @@ templates: original_task: true name: i_am_hesitating reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 4c4fdeff-45c7-4baa-a889-f56a5430c638: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + id: 4c4fdeff-45c7-4baa-a889-f56a5430c638 jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% else %} As a consequence... {% endif %} @@ -73,12 +73,12 @@ templates: original_task: true name: plausible_alternatives reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 52344c8c-2aaf-4593-8b85-30b6bac14318: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 52344c8c-2aaf-4593-8b85-30b6bac14318 + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -88,9 +88,9 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + 5a14b4e9-7b61-41f4-b05c-b823752ea693: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 5a14b4e9-7b61-41f4-b05c-b823752ea693 jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,11 +103,11 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + 6cf55f8c-bc20-47ba-97b4-d958dcb99e53: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + id: 6cf55f8c-bc20-47ba-97b4-d958dcb99e53 jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true @@ -118,9 +118,9 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + b5388b41-363a-4a33-b82e-175497f884f4: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + id: b5388b41-363a-4a33-b82e-175497f884f4 jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -133,12 +133,12 @@ templates: original_task: true name: "\u2026which may be caused by" reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + c28c7dbc-615a-400e-9099-8df5a0ab1ef5: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + id: c28c7dbc-615a-400e-9099-8df5a0ab1ef5 jinja: 'Pick the more likely continuation to the following sentence: - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: {% endif %} - {{choice1}} @@ -153,13 +153,13 @@ templates: original_task: true name: more likely reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + c717cdaa-0b19-4288-9635-2a783bc87acf: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af + id: c717cdaa-0b19-4288-9635-2a783bc87acf jinja: '{{ premise }} - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -174,9 +174,9 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + cac78748-4b44-45e9-ad52-32bccaf286c6: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + id: cac78748-4b44-45e9-ad52-32bccaf286c6 jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -189,9 +189,9 @@ templates: original_task: true name: "\u2026why? C1 or C2" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + fcc668f5-15f5-41ee-8f1e-d62476b18939: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 + id: fcc668f5-15f5-41ee-8f1e-d62476b18939 jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif %} @@ -208,4 +208,4 @@ templates: - Accuracy original_task: true name: choose - reference: '' + reference: '' \ No newline at end of file diff --git a/promptsource/templates/xcopa/ta/templates.yaml b/promptsource/templates/xcopa/ta/templates.yaml index 5d5d95b98..c9d620cc0 100644 --- a/promptsource/templates/xcopa/ta/templates.yaml +++ b/promptsource/templates/xcopa/ta/templates.yaml @@ -1,10 +1,10 @@ dataset: xcopa subset: ta templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 2a1a3df1-97c5-43b5-8ee4-3cb7a25c5e8e: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Tamil sentence: + id: 2a1a3df1-97c5-43b5-8ee4-3cb7a25c5e8e + jinja: 'Exercise: choose the most plausible alternative. {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif @@ -22,9 +22,9 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 2a57331a-646c-45ab-ac8c-8976aa1a08c9: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + id: 2a57331a-646c-45ab-ac8c-8976aa1a08c9 jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -37,9 +37,9 @@ templates: original_task: true name: "\u2026What could happen next, C1 or C2?" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 34e9a83d-dec9-4541-a6f3-fd641e476631: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + id: 34e9a83d-dec9-4541-a6f3-fd641e476631 jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ @@ -53,9 +53,9 @@ templates: original_task: true name: i_am_hesitating reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 45503d4d-2cc4-461c-a41c-5c5924a4c63c: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + id: 45503d4d-2cc4-461c-a41c-5c5924a4c63c jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% else %} As a consequence... {% endif %} @@ -73,12 +73,12 @@ templates: original_task: true name: plausible_alternatives reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 50ce0772-7e29-462e-a0f5-0b5172b3b785: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 50ce0772-7e29-462e-a0f5-0b5172b3b785 + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -88,9 +88,9 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + 694f0d12-c333-4a95-90f4-e72dae5d54b3: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 694f0d12-c333-4a95-90f4-e72dae5d54b3 jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,11 +103,11 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + b3b94d15-4608-43f1-9b6e-c577ca28b267: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + id: b3b94d15-4608-43f1-9b6e-c577ca28b267 jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true @@ -118,9 +118,9 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + b552e0a9-1f0b-41ef-95ee-149c2542fd7d: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + id: b552e0a9-1f0b-41ef-95ee-149c2542fd7d jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -133,12 +133,12 @@ templates: original_task: true name: "\u2026which may be caused by" reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + bb5dc8aa-ef3d-4a9e-bab2-7c80cc2d64da: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + id: bb5dc8aa-ef3d-4a9e-bab2-7c80cc2d64da jinja: 'Pick the more likely continuation to the following sentence: - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: {% endif %} - {{choice1}} @@ -153,13 +153,13 @@ templates: original_task: true name: more likely reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + da476901-0e1d-47ea-b97b-d6075190db32: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af + id: da476901-0e1d-47ea-b97b-d6075190db32 jinja: '{{ premise }} - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -174,9 +174,9 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + f89bdc34-7b27-43f4-b54d-0868e6540240: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + id: f89bdc34-7b27-43f4-b54d-0868e6540240 jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -189,7 +189,7 @@ templates: original_task: true name: "\u2026why? C1 or C2" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + fd13aa2f-ff7b-41fb-9cc4-842f091394c2: !Template answer_choices: '{{choice1}} ||| {{choice2}}' id: f32348cd-d3cb-4619-87b9-e24f99c78567 jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... @@ -209,3 +209,4 @@ templates: original_task: true name: choose reference: '' + From e02074005ea2289c75a65ef867ae40ec0f9037fa Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 17:09:08 -0400 Subject: [PATCH 04/17] regenerate ids (5/6) --- .../templates/xcopa/zh/templates.yaml | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/promptsource/templates/xcopa/zh/templates.yaml b/promptsource/templates/xcopa/zh/templates.yaml index 4d5792f78..dbf4f482f 100644 --- a/promptsource/templates/xcopa/zh/templates.yaml +++ b/promptsource/templates/xcopa/zh/templates.yaml @@ -1,10 +1,10 @@ dataset: xcopa subset: zh templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 059e17fc-11f0-4072-bbdc-d4667f77466b: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Chinese sentence: + id: 059e17fc-11f0-4072-bbdc-d4667f77466b + jinja: 'Exercise: choose the most plausible alternative. {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif @@ -22,9 +22,9 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 1e0fc116-2741-460a-aefb-a8af579c1c57: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + id: 1e0fc116-2741-460a-aefb-a8af579c1c57 jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -37,9 +37,9 @@ templates: original_task: true name: "\u2026What could happen next, C1 or C2?" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 3814efad-35ff-4c42-a666-084514d6356b: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + id: 3814efad-35ff-4c42-a666-084514d6356b jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ @@ -53,9 +53,9 @@ templates: original_task: true name: i_am_hesitating reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 44a40995-73c8-4b15-b830-f5fd55473a59: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + id: 44a40995-73c8-4b15-b830-f5fd55473a59 jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% else %} As a consequence... {% endif %} @@ -73,12 +73,12 @@ templates: original_task: true name: plausible_alternatives reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 48888279-affe-423a-80c8-51937f872ac8: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 48888279-affe-423a-80c8-51937f872ac8 + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -88,9 +88,9 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + 4b7be3a0-a8f4-4289-9a39-297ce5ec0dae: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 4b7be3a0-a8f4-4289-9a39-297ce5ec0dae jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,11 +103,11 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + b3f10790-e8cf-4d05-b974-fd58b981ea26: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + id: b3f10790-e8cf-4d05-b974-fd58b981ea26 jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true @@ -118,9 +118,9 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + b6bd0360-3b39-4ae3-b837-b3fe8e7aefbf: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + id: b6bd0360-3b39-4ae3-b837-b3fe8e7aefbf jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -133,12 +133,12 @@ templates: original_task: true name: "\u2026which may be caused by" reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + c3b01f23-b69a-4a31-9258-08475490c4e4: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + id: c3b01f23-b69a-4a31-9258-08475490c4e4 jinja: 'Pick the more likely continuation to the following sentence: - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: {% endif %} - {{choice1}} @@ -153,13 +153,13 @@ templates: original_task: true name: more likely reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + cbae3be6-a416-4dfe-bbf9-6dbca7f2e542: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af + id: cbae3be6-a416-4dfe-bbf9-6dbca7f2e542 jinja: '{{ premise }} - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -174,9 +174,9 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + dfa40a8a-cfe1-4695-b566-a5d83c98d630: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + id: dfa40a8a-cfe1-4695-b566-a5d83c98d630 jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -189,9 +189,9 @@ templates: original_task: true name: "\u2026why? C1 or C2" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + f42c13d4-8305-4499-b73c-6e20ed83569c: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 + id: f42c13d4-8305-4499-b73c-6e20ed83569c jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif %} @@ -209,3 +209,4 @@ templates: original_task: true name: choose reference: '' + From 8511070b44f36abd38bbc13bf4c159c78a1e8515 Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 17:12:26 -0400 Subject: [PATCH 05/17] regenerate ids (6/6) --- .../templates/xcopa/vi/templates.yaml | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/promptsource/templates/xcopa/vi/templates.yaml b/promptsource/templates/xcopa/vi/templates.yaml index 1d78885dc..81ccd36f4 100644 --- a/promptsource/templates/xcopa/vi/templates.yaml +++ b/promptsource/templates/xcopa/vi/templates.yaml @@ -1,10 +1,10 @@ dataset: xcopa subset: vi templates: - 0edd8660-f299-4819-a5ac-633c11177228: !Template + 1804535c-1953-4dbb-960e-c41a634a8dc7: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 0edd8660-f299-4819-a5ac-633c11177228 - jinja: 'Exercise: Choose the most plausible completion to this Vietnamese sentence: + id: 1804535c-1953-4dbb-960e-c41a634a8dc7 + jinja: 'Exercise: choose the most plausible alternative. {{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif @@ -22,9 +22,9 @@ templates: original_task: true name: exercise reference: '' - 150789fe-e309-47a1-82c9-0a4dc2c6b12b: !Template + 2bd073ac-1b89-48ea-b401-0343543e5675: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 150789fe-e309-47a1-82c9-0a4dc2c6b12b + id: 2bd073ac-1b89-48ea-b401-0343543e5675 jinja: "{% if question == \"effect\" %} \n{{ premise }} What could happen next,\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -37,9 +37,9 @@ templates: original_task: true name: "\u2026What could happen next, C1 or C2?" reference: '' - 4d879cbe-2fd7-424a-9d78-3f5200313fba: !Template + 362a97b2-0113-40b0-9928-7aaac4138bb5: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 4d879cbe-2fd7-424a-9d78-3f5200313fba + id: 362a97b2-0113-40b0-9928-7aaac4138bb5 jinja: "{{ premise }} \n\nI am hesitating between two options. Help me choose\ \ the more likely {% if question == \"cause\" %} cause: {% else %} effect: {%\ \ endif %}\n- {{choice1}}\n- {{choice2}} ||| {% if label != -1 %}{{ answer_choices[label]\ @@ -53,9 +53,9 @@ templates: original_task: true name: i_am_hesitating reference: '' - 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d: !Template + 38f5300e-27e2-43b8-a9d5-14be72a33dcb: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 66ea075e-4d03-4a78-b1fa-9a5228cf0c9d + id: 38f5300e-27e2-43b8-a9d5-14be72a33dcb jinja: '{{ premise }} {% if question == "cause" %} This happened because... {% else %} As a consequence... {% endif %} @@ -73,12 +73,12 @@ templates: original_task: true name: plausible_alternatives reference: '' - 744047dc-1298-45a2-8d68-d67e3f834ded: !Template - answer_choices: '{{choice1 }} ||| {{choice2}}' - id: 744047dc-1298-45a2-8d68-d67e3f834ded - jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }}{% - if question == "cause" %}because {% else %} so {% endif %} ||| {% if label != - -1 %}{{ answer_choices[label] }}{% endif %}' + 649a8094-b640-4d63-9cd1-ef286312c564: !Template + answer_choices: '{{choice1}} ||| {{choice2}}' + id: 649a8094-b640-4d63-9cd1-ef286312c564 + jinja: '"{{ answer_choices[0] }}" or "{{ answer_choices[1] }}"? {{ premise }} + {% if question == "cause" %} because {% else %} so {% endif %} ||| {% if label + != -1 %}{{ answer_choices[label] }}{% endif %}' metadata: !TemplateMetadata choices_in_prompt: true languages: @@ -88,9 +88,9 @@ templates: original_task: true name: "C1 or C2? premise, so/because\u2026" reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." - 84da62c2-9440-4cfc-bdd4-d70c65e33a82: !Template + 7871d0ec-ee9c-4db6-ae8c-83d20acc9e56: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 84da62c2-9440-4cfc-bdd4-d70c65e33a82 + id: 7871d0ec-ee9c-4db6-ae8c-83d20acc9e56 jinja: "{% if question == \"effect\" %} \n{{ premise }} As a result, \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -103,11 +103,11 @@ templates: original_task: true name: "\u2026As a result, C1 or C2?" reference: '' - 8ce80f8a-239e-4393-892c-f63dbb0d9929: !Template + 80130200-be7b-4f2f-af3b-758d0d3b35b1: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8ce80f8a-239e-4393-892c-f63dbb0d9929 + id: 80130200-be7b-4f2f-af3b-758d0d3b35b1 jinja: "{{ premise }} \n\nWhat's the best option?\n- {{choice1}}\n- {{choice2}}\n\ - \nWe are looking for {% if question == \"cause\" %}a cause {% else %}an effect\ + \nWe are looking for {% if question == \"cause\" %} a cause {% else %} an effect\ \ {% endif %}\n||| {% if label != -1 %}{{answer_choices[label]}}{%endif%}" metadata: !TemplateMetadata choices_in_prompt: true @@ -118,9 +118,9 @@ templates: original_task: true name: best_option reference: '' - 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb: !Template + 8db169df-6a91-4c0a-9629-be7178e7618e: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: 8cf2ba73-aee5-4651-b5d4-b1b88afe4abb + id: 8db169df-6a91-4c0a-9629-be7178e7618e jinja: "{% if question == \"cause\" %} \n{{ premise }} Which may be caused by\ \ \"{{ answer_choices[0] }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label\ \ != -1 %}{{ answer_choices[label] }}{%endif%}\n{% endif %}" @@ -133,12 +133,12 @@ templates: original_task: true name: "\u2026which may be caused by" reference: '' - a1f9951e-2b6b-4530-9636-9cdf4c1658c5: !Template + 90564726-ca08-4812-a038-4b1f54fbcff9: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a1f9951e-2b6b-4530-9636-9cdf4c1658c5 + id: 90564726-ca08-4812-a038-4b1f54fbcff9 jinja: 'Pick the more likely continuation to the following sentence: - {{ premise }}{% if question == "cause" %} As a result of: {% else %} As a consequence: + {{ premise }} {% if question == "cause" %} as a result of: {% else %} as a consequence: {% endif %} - {{choice1}} @@ -153,13 +153,13 @@ templates: original_task: true name: more likely reference: '' - a61d8c21-da25-47bf-b5fe-14a8edd650af: !Template + a506f224-721f-45dc-bb0f-46d9d6193e6a: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a61d8c21-da25-47bf-b5fe-14a8edd650af + id: a506f224-721f-45dc-bb0f-46d9d6193e6a jinja: '{{ premise }} - Select the most plausible {% if question == "cause" %}cause: {% else %}effect: + Select the most plausible {% if question == "cause" %} cause: {% else %} effect: {% endif %} - {{choice1}} @@ -174,9 +174,9 @@ templates: original_task: true name: cause_effect reference: '' - a8bf11c3-bea2-45ba-a533-957d8bee5e2e: !Template + d0bb6c48-2e2c-4606-bd2b-f1eca7fa6f37: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: a8bf11c3-bea2-45ba-a533-957d8bee5e2e + id: d0bb6c48-2e2c-4606-bd2b-f1eca7fa6f37 jinja: "{% if question == \"cause\" %} \n{{ premise }} Why? \"{{ answer_choices[0]\ \ }}\" or \"{{ answer_choices[1] }}\"? ||| {% if label != -1 %}{{ answer_choices[label]\ \ }}{%endif%}\n{% endif %}" @@ -189,9 +189,9 @@ templates: original_task: true name: "\u2026why? C1 or C2" reference: '' - f32348cd-d3cb-4619-87b9-e24f99c78567: !Template + fb619264-6ad3-4b42-9eda-c87832c4e31c: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 + id: fb619264-6ad3-4b42-9eda-c87832c4e31c jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif %} From cc65cf38d702021abe3f68723b071e0674c9d18a Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 17:21:41 -0400 Subject: [PATCH 06/17] fix typo --- promptsource/templates/xcopa/it/templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promptsource/templates/xcopa/it/templates.yaml b/promptsource/templates/xcopa/it/templates.yaml index 99141407d..4b1d37392 100644 --- a/promptsource/templates/xcopa/it/templates.yaml +++ b/promptsource/templates/xcopa/it/templates.yaml @@ -134,7 +134,7 @@ templates: name: "\u2026which may be caused by" reference: '' b90379c6-98d6-436e-a0db-466ad0f5a1c1: !Template - answer_choices: '{{choice1}} ||| {{choice2}}'s + answer_choices: '{{choice1}} ||| {{choice2}}' id: b90379c6-98d6-436e-a0db-466ad0f5a1c1 jinja: 'Pick the more likely continuation to the following sentence: From 670e6ad7aa1883e59c896f203749d3d118166c4f Mon Sep 17 00:00:00 2001 From: haileyschoelkopf Date: Fri, 15 Jul 2022 17:31:37 -0400 Subject: [PATCH 07/17] fix last keyerror? --- promptsource/templates/xcopa/ta/templates.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promptsource/templates/xcopa/ta/templates.yaml b/promptsource/templates/xcopa/ta/templates.yaml index c9d620cc0..bdd79f7f2 100644 --- a/promptsource/templates/xcopa/ta/templates.yaml +++ b/promptsource/templates/xcopa/ta/templates.yaml @@ -191,7 +191,7 @@ templates: reference: '' fd13aa2f-ff7b-41fb-9cc4-842f091394c2: !Template answer_choices: '{{choice1}} ||| {{choice2}}' - id: f32348cd-d3cb-4619-87b9-e24f99c78567 + id: fd13aa2f-ff7b-41fb-9cc4-842f091394c2 jinja: '{{ premise }} {% if question == "cause" %} because... {% else %} so... {% endif %} From c56ce86325a63b4f94fbb19a66d760af8ab94a0b Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Sun, 17 Jul 2022 23:09:20 +0200 Subject: [PATCH 08/17] Add xwinograd/en --- promptsource/templates.py | 2 +- .../Muennighoff/xwinograd/en/templates.yaml | 110 ++++++++++++++++++ 2 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 promptsource/templates/Muennighoff/xwinograd/en/templates.yaml diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..467a0f646 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,7 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti", "Muennighoff"} # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml new file mode 100644 index 000000000..e30dd5ac3 --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 28d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 28d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' From d164dbf35a81db772245d4118f5ac8d373fed8d4 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Sun, 17 Jul 2022 23:29:05 +0200 Subject: [PATCH 09/17] Duplicate template --- .../Muennighoff/xwinograd/en/templates.yaml | 4 +- .../Muennighoff/xwinograd/fr/templates.yaml | 110 ++++++++++++++++++ .../Muennighoff/xwinograd/pt/templates.yaml | 110 ++++++++++++++++++ .../Muennighoff/xwinograd/zh/templates.yaml | 110 ++++++++++++++++++ 4 files changed, 332 insertions(+), 2 deletions(-) create mode 100644 promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml create mode 100644 promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml create mode 100644 promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml diff --git a/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml index e30dd5ac3..f0f34dccb 100644 --- a/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml +++ b/promptsource/templates/Muennighoff/xwinograd/en/templates.yaml @@ -45,7 +45,7 @@ templates: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template answer_choices: True ||| False id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa - jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? {{sentence}}||| @@ -62,7 +62,7 @@ templates: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template answer_choices: '{{ option1 }} ||| {{ option2 }}' id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e - jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif %}' metadata: !TemplateMetadata diff --git a/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml new file mode 100644 index 000000000..f0f34dccb --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 28d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 28d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' diff --git a/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml new file mode 100644 index 000000000..f0f34dccb --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: en +templates: + 28d31908-4aee-4545-aff2-7528cbf39197: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 28d31908-4aee-4545-aff2-7528cbf39197 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + answer_choices: True ||| False + id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' diff --git a/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml new file mode 100644 index 000000000..c911d8cb2 --- /dev/null +++ b/promptsource/templates/Muennighoff/xwinograd/zh/templates.yaml @@ -0,0 +1,110 @@ +dataset: Muennighoff/xwinograd +subset: zh +templates: + 2e10b0b9-b57e-4bd0-b8cc-cbdd5dcd7551: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 2e10b0b9-b57e-4bd0-b8cc-cbdd5dcd7551 + jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ + \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ + \ else %} {{ option2 }} {% endif %}" + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: Replace + reference: '' + 33f50367-0615-45a0-883a-05b6ff8c0e2f: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: 33f50367-0615-45a0-883a-05b6ff8c0e2f + jinja: 'Fill in the _ in the below sentence: + + {{sentence}} + + + Choices: + + - {{ option1 }} + + - {{ option2 }} + + + Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% + endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: fill in the blank + reference: '' + 37767236-f6e6-4a57-9deb-c0a4c073e56d: !Template + answer_choices: True ||| False + id: 37767236-f6e6-4a57-9deb-c0a4c073e56d + jinja: 'The _ in the sentence below refers to {{option1}}. True or False? + + {{sentence}}||| + + {{answer_choices[answer|int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: false + name: True or False + reference: '' + 63f2ab14-d78b-4e83-a3f9-186cb6f220c0: !Template + answer_choices: '{{ option1 }} ||| {{ option2 }}' + id: 63f2ab14-d78b-4e83-a3f9-186cb6f220c0 + jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} + or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 + }} {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: does underscore refer to + reference: '' + aec099a7-da35-4438-aa7c-250651959048: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: aec099a7-da35-4438-aa7c-250651959048 + jinja: '{{sentence}} + + What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 + }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif + %}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: underscore refer to + reference: '' + fa9d56aa-b236-44f9-92c1-595784f57243: !Template + answer_choices: '{{option1}} ||| {{option2}}' + id: fa9d56aa-b236-44f9-92c1-595784f57243 + jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? + + {{sentence}}||| + + {{answer_choices[answer | int - 1]}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + original_task: true + name: stand for + reference: '' From 0bfade4a71cd9b90b744a1b17e0dff04c8e0da49 Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 18 Jul 2022 08:56:44 +0200 Subject: [PATCH 10/17] Format --- promptsource/templates.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/promptsource/templates.py b/promptsource/templates.py index 467a0f646..789e91cc8 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,18 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti", "Muennighoff"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "Muennighoff", +} # These are the metrics with which templates can be tagged METRICS = { From c80db698c2b8dc43ac7474253461dcefa84ce45b Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 10:03:32 +0000 Subject: [PATCH 11/17] wip -test --- test/test_templates.py | 178 ++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/test/test_templates.py b/test/test_templates.py index 20f774903..e13685c71 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -37,92 +37,92 @@ def test_uuids(): all_uuids[uuid] = (dataset_name, subset_name) -@pytest.mark.parametrize("dataset", template_collection.keys) -def test_dataset(dataset): - """ - Validates all the templates in the repository with simple syntactic checks: - 0. Are all templates parsable YAML? - 1. Do all templates parse in Jinja and are all referenced variables in the dataset schema? - 2. Does the template contain a prompt/output separator "|||" ? - 3. Are all names and templates within a data (sub)set unique? - 4. Is the YAML dictionary properly formatted? - 5. Is the UUID valid? - - :param dataset: (dataset_name, subset_name) pair to test - - """ - dataset_name, subset_name = dataset - - # Loads dataset information - tries = 0 - max_tries = 3 - while True: - try: - builder_instance = get_dataset_builder(dataset_name, subset_name) - break - except ConnectionError as e: - if tries < max_tries: - time.sleep(2) - tries += 1 - else: - raise e - - has_features = builder_instance.info.features is not None - if has_features: - features = builder_instance.info.features.keys() - features = set([feature.replace("-", "_") for feature in features]) - - # Initializes sets for checking uniqueness among templates - template_name_set = set() - template_jinja_set = set() - - # Iterates over each template for current data (sub)set - dataset_templates = template_collection.get_dataset(dataset_name, subset_name) - any_original = False - for template_name in dataset_templates.all_template_names: - template = dataset_templates[template_name] - any_original = any_original or template.metadata.original_task - # Check 1: Jinja and all features valid? - try: - parse = env.parse(template.jinja) - except TemplateError as e: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} failed to parse.") from e - - variables = meta.find_undeclared_variables(parse) - for variable in variables: - if has_features and variable not in features and variable != "answer_choices": - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has unrecognized variable {variable}.") - - # Check 2: Prompt/output separator present? - if "|||" not in template.jinja: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has no input/target separator.") - - # Check 3: Unique names and templates? - if template.get_name() in template_name_set: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has duplicate name.") - - if template.jinja in template_jinja_set: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has duplicate definition.") - - template_name_set.add(template.get_name()) - template_jinja_set.add(template.jinja) - - # Check 4: Is the YAML dictionary properly formatted? - try: - if dataset_templates.templates[template.get_id()] != template: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has wrong YAML key.") - except KeyError as e: - raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " - f"with uuid {template.get_id()} has wrong YAML key.") from e - - # Check 5: Is the UUID valid? - UUID(template.get_id()) - - # Turned off for now until we fix. - #assert any_original, "There must be at least one original task template for each dataset" +# @pytest.mark.parametrize("dataset", template_collection.keys) +# def test_dataset(dataset): +# """ +# Validates all the templates in the repository with simple syntactic checks: +# 0. Are all templates parsable YAML? +# 1. Do all templates parse in Jinja and are all referenced variables in the dataset schema? +# 2. Does the template contain a prompt/output separator "|||" ? +# 3. Are all names and templates within a data (sub)set unique? +# 4. Is the YAML dictionary properly formatted? +# 5. Is the UUID valid? + +# :param dataset: (dataset_name, subset_name) pair to test + +# """ +# dataset_name, subset_name = dataset + +# # Loads dataset information +# tries = 0 +# max_tries = 3 +# while True: +# try: +# builder_instance = get_dataset_builder(dataset_name, subset_name) +# break +# except ConnectionError as e: +# if tries < max_tries: +# time.sleep(2) +# tries += 1 +# else: +# raise e + +# has_features = builder_instance.info.features is not None +# if has_features: +# features = builder_instance.info.features.keys() +# features = set([feature.replace("-", "_") for feature in features]) + +# # Initializes sets for checking uniqueness among templates +# template_name_set = set() +# template_jinja_set = set() + +# # Iterates over each template for current data (sub)set +# dataset_templates = template_collection.get_dataset(dataset_name, subset_name) +# any_original = False +# for template_name in dataset_templates.all_template_names: +# template = dataset_templates[template_name] +# any_original = any_original or template.metadata.original_task +# # Check 1: Jinja and all features valid? +# try: +# parse = env.parse(template.jinja) +# except TemplateError as e: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} failed to parse.") from e + +# variables = meta.find_undeclared_variables(parse) +# for variable in variables: +# if has_features and variable not in features and variable != "answer_choices": +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has unrecognized variable {variable}.") + +# # Check 2: Prompt/output separator present? +# if "|||" not in template.jinja: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has no input/target separator.") + +# # Check 3: Unique names and templates? +# if template.get_name() in template_name_set: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has duplicate name.") + +# if template.jinja in template_jinja_set: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has duplicate definition.") + +# template_name_set.add(template.get_name()) +# template_jinja_set.add(template.jinja) + +# # Check 4: Is the YAML dictionary properly formatted? +# try: +# if dataset_templates.templates[template.get_id()] != template: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has wrong YAML key.") +# except KeyError as e: +# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " +# f"with uuid {template.get_id()} has wrong YAML key.") from e + +# # Check 5: Is the UUID valid? +# UUID(template.get_id()) + +# # Turned off for now until we fix. +# #assert any_original, "There must be at least one original task template for each dataset" From 12ef8de7e8e7241836d76d1f1074529a62981502 Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 10:09:20 +0000 Subject: [PATCH 12/17] wip - test --- test/test_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_templates.py b/test/test_templates.py index e13685c71..fa54ef358 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -28,6 +28,7 @@ def test_uuids(): template = dataset_templates[template_name] uuid = template.get_id() + print(dataset_name, subset_name, uuid) if uuid in all_uuids: raise ValueError(f"Template {template_name} for dataset {dataset_name}/{subset_name} " From 35e29a840f30daf319d5c92276bd6eb9477d37ed Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 10:23:51 +0000 Subject: [PATCH 13/17] wip - most stupid --- test/test_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_templates.py b/test/test_templates.py index fa54ef358..172787bb7 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -28,6 +28,7 @@ def test_uuids(): template = dataset_templates[template_name] uuid = template.get_id() + raise ValueError(3) print(dataset_name, subset_name, uuid) if uuid in all_uuids: From 4d5c50003276b46004b4ff6982763521776a5e9b Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 10:26:09 +0000 Subject: [PATCH 14/17] wip - test --- test/test_templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_templates.py b/test/test_templates.py index 172787bb7..fa54ef358 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -28,7 +28,6 @@ def test_uuids(): template = dataset_templates[template_name] uuid = template.get_id() - raise ValueError(3) print(dataset_name, subset_name, uuid) if uuid in all_uuids: From 62ee1a8683496f0cfbcf03ad950b7f2193cea011 Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 10:29:54 +0000 Subject: [PATCH 15/17] de --- test/test_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_templates.py b/test/test_templates.py index fa54ef358..336d2c108 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -29,6 +29,7 @@ def test_uuids(): uuid = template.get_id() print(dataset_name, subset_name, uuid) + assert uuid not in all_uuids if uuid in all_uuids: raise ValueError(f"Template {template_name} for dataset {dataset_name}/{subset_name} " From 1d55c8dc70478520aeb1e838822442e4c2633fdf Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Mon, 18 Jul 2022 12:16:54 +0000 Subject: [PATCH 16/17] re-clean it - have not make it work yet --- test/test_templates.py | 180 ++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 91 deletions(-) diff --git a/test/test_templates.py b/test/test_templates.py index 336d2c108..20f774903 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -28,8 +28,6 @@ def test_uuids(): template = dataset_templates[template_name] uuid = template.get_id() - print(dataset_name, subset_name, uuid) - assert uuid not in all_uuids if uuid in all_uuids: raise ValueError(f"Template {template_name} for dataset {dataset_name}/{subset_name} " @@ -39,92 +37,92 @@ def test_uuids(): all_uuids[uuid] = (dataset_name, subset_name) -# @pytest.mark.parametrize("dataset", template_collection.keys) -# def test_dataset(dataset): -# """ -# Validates all the templates in the repository with simple syntactic checks: -# 0. Are all templates parsable YAML? -# 1. Do all templates parse in Jinja and are all referenced variables in the dataset schema? -# 2. Does the template contain a prompt/output separator "|||" ? -# 3. Are all names and templates within a data (sub)set unique? -# 4. Is the YAML dictionary properly formatted? -# 5. Is the UUID valid? - -# :param dataset: (dataset_name, subset_name) pair to test - -# """ -# dataset_name, subset_name = dataset - -# # Loads dataset information -# tries = 0 -# max_tries = 3 -# while True: -# try: -# builder_instance = get_dataset_builder(dataset_name, subset_name) -# break -# except ConnectionError as e: -# if tries < max_tries: -# time.sleep(2) -# tries += 1 -# else: -# raise e - -# has_features = builder_instance.info.features is not None -# if has_features: -# features = builder_instance.info.features.keys() -# features = set([feature.replace("-", "_") for feature in features]) - -# # Initializes sets for checking uniqueness among templates -# template_name_set = set() -# template_jinja_set = set() - -# # Iterates over each template for current data (sub)set -# dataset_templates = template_collection.get_dataset(dataset_name, subset_name) -# any_original = False -# for template_name in dataset_templates.all_template_names: -# template = dataset_templates[template_name] -# any_original = any_original or template.metadata.original_task -# # Check 1: Jinja and all features valid? -# try: -# parse = env.parse(template.jinja) -# except TemplateError as e: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} failed to parse.") from e - -# variables = meta.find_undeclared_variables(parse) -# for variable in variables: -# if has_features and variable not in features and variable != "answer_choices": -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has unrecognized variable {variable}.") - -# # Check 2: Prompt/output separator present? -# if "|||" not in template.jinja: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has no input/target separator.") - -# # Check 3: Unique names and templates? -# if template.get_name() in template_name_set: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has duplicate name.") - -# if template.jinja in template_jinja_set: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has duplicate definition.") - -# template_name_set.add(template.get_name()) -# template_jinja_set.add(template.jinja) - -# # Check 4: Is the YAML dictionary properly formatted? -# try: -# if dataset_templates.templates[template.get_id()] != template: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has wrong YAML key.") -# except KeyError as e: -# raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " -# f"with uuid {template.get_id()} has wrong YAML key.") from e - -# # Check 5: Is the UUID valid? -# UUID(template.get_id()) - -# # Turned off for now until we fix. -# #assert any_original, "There must be at least one original task template for each dataset" +@pytest.mark.parametrize("dataset", template_collection.keys) +def test_dataset(dataset): + """ + Validates all the templates in the repository with simple syntactic checks: + 0. Are all templates parsable YAML? + 1. Do all templates parse in Jinja and are all referenced variables in the dataset schema? + 2. Does the template contain a prompt/output separator "|||" ? + 3. Are all names and templates within a data (sub)set unique? + 4. Is the YAML dictionary properly formatted? + 5. Is the UUID valid? + + :param dataset: (dataset_name, subset_name) pair to test + + """ + dataset_name, subset_name = dataset + + # Loads dataset information + tries = 0 + max_tries = 3 + while True: + try: + builder_instance = get_dataset_builder(dataset_name, subset_name) + break + except ConnectionError as e: + if tries < max_tries: + time.sleep(2) + tries += 1 + else: + raise e + + has_features = builder_instance.info.features is not None + if has_features: + features = builder_instance.info.features.keys() + features = set([feature.replace("-", "_") for feature in features]) + + # Initializes sets for checking uniqueness among templates + template_name_set = set() + template_jinja_set = set() + + # Iterates over each template for current data (sub)set + dataset_templates = template_collection.get_dataset(dataset_name, subset_name) + any_original = False + for template_name in dataset_templates.all_template_names: + template = dataset_templates[template_name] + any_original = any_original or template.metadata.original_task + # Check 1: Jinja and all features valid? + try: + parse = env.parse(template.jinja) + except TemplateError as e: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} failed to parse.") from e + + variables = meta.find_undeclared_variables(parse) + for variable in variables: + if has_features and variable not in features and variable != "answer_choices": + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has unrecognized variable {variable}.") + + # Check 2: Prompt/output separator present? + if "|||" not in template.jinja: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has no input/target separator.") + + # Check 3: Unique names and templates? + if template.get_name() in template_name_set: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has duplicate name.") + + if template.jinja in template_jinja_set: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has duplicate definition.") + + template_name_set.add(template.get_name()) + template_jinja_set.add(template.jinja) + + # Check 4: Is the YAML dictionary properly formatted? + try: + if dataset_templates.templates[template.get_id()] != template: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has wrong YAML key.") + except KeyError as e: + raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " + f"with uuid {template.get_id()} has wrong YAML key.") from e + + # Check 5: Is the UUID valid? + UUID(template.get_id()) + + # Turned off for now until we fix. + #assert any_original, "There must be at least one original task template for each dataset" From fcc9b8c65de48ad1ce330d462d6642416f02b34a Mon Sep 17 00:00:00 2001 From: Muennighoff Date: Mon, 18 Jul 2022 14:21:08 +0200 Subject: [PATCH 17/17] Change IDs --- .../Muennighoff/xwinograd/fr/templates.yaml | 24 +++++++++---------- .../Muennighoff/xwinograd/pt/templates.yaml | 24 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml index f0f34dccb..5c038558c 100644 --- a/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml +++ b/promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml @@ -1,9 +1,9 @@ dataset: Muennighoff/xwinograd subset: en templates: - 28d31908-4aee-4545-aff2-7528cbf39197: !Template + 38d31908-4aee-4545-aff2-7528cbf39197: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: 28d31908-4aee-4545-aff2-7528cbf39197 + id: 38d31908-4aee-4545-aff2-7528cbf39197 jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ \ else %} {{ option2 }} {% endif %}" @@ -16,9 +16,9 @@ templates: original_task: true name: Replace reference: '' - 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + 60ce5113-882f-4a9d-b21d-8d98b4644295: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + id: 60ce5113-882f-4a9d-b21d-8d98b4644295 jinja: 'Fill in the _ in the below sentence: {{sentence}} @@ -42,9 +42,9 @@ templates: original_task: true name: fill in the blank reference: '' - 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + 8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template answer_choices: True ||| False - id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa jinja: 'The _ in the sentence below refers to {{option1}}. True or False? {{sentence}}||| @@ -59,9 +59,9 @@ templates: original_task: false name: True or False reference: '' - 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template answer_choices: '{{ option1 }} ||| {{ option2 }}' - id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif %}' @@ -74,9 +74,9 @@ templates: original_task: true name: does underscore refer to reference: '' - bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202 jinja: '{{sentence}} What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 @@ -91,9 +91,9 @@ templates: original_task: true name: underscore refer to reference: '' - ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? {{sentence}}||| diff --git a/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml index f0f34dccb..5c038558c 100644 --- a/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml +++ b/promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml @@ -1,9 +1,9 @@ dataset: Muennighoff/xwinograd subset: en templates: - 28d31908-4aee-4545-aff2-7528cbf39197: !Template + 38d31908-4aee-4545-aff2-7528cbf39197: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: 28d31908-4aee-4545-aff2-7528cbf39197 + id: 38d31908-4aee-4545-aff2-7528cbf39197 jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\ \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\ \ else %} {{ option2 }} {% endif %}" @@ -16,9 +16,9 @@ templates: original_task: true name: Replace reference: '' - 50ce5113-882f-4a9d-b21d-8d98b4644295: !Template + 60ce5113-882f-4a9d-b21d-8d98b4644295: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: 50ce5113-882f-4a9d-b21d-8d98b4644295 + id: 60ce5113-882f-4a9d-b21d-8d98b4644295 jinja: 'Fill in the _ in the below sentence: {{sentence}} @@ -42,9 +42,9 @@ templates: original_task: true name: fill in the blank reference: '' - 7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template + 8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template answer_choices: True ||| False - id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa + id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa jinja: 'The _ in the sentence below refers to {{option1}}. True or False? {{sentence}}||| @@ -59,9 +59,9 @@ templates: original_task: false name: True or False reference: '' - 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template + 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template answer_choices: '{{ option1 }} ||| {{ option2 }}' - id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e + id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }} or {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif %}' @@ -74,9 +74,9 @@ templates: original_task: true name: does underscore refer to reference: '' - bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template + cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202 + id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202 jinja: '{{sentence}} What does the _ in the above sentence refer to? {{ option1 }} or {{ option2 @@ -91,9 +91,9 @@ templates: original_task: true name: underscore refer to reference: '' - ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template + fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template answer_choices: '{{option1}} ||| {{option2}}' - id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d + id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}? {{sentence}}|||