From a3ed6a3fc0fa668fa3f5d0cdb5d355b870a8c082 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Fri, 22 Jul 2022 19:37:14 +0200
Subject: [PATCH 01/36] Add cp complexity

---
 promptsource/templates.py                     | 16 ++++++-
 .../codeparrot--codecomplex/templates.yaml    | 45 +++++++++++++++++++
 2 files changed, 60 insertions(+), 1 deletion(-)
 create mode 100644 promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 2b9eb341e..943f8611e 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -27,7 +27,21 @@
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
 
-INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"}
+INCLUDED_USERS = {
+    "Zaid",
+    "craffel",
+    "GEM",
+    "aps",
+    "khalidalt",
+    "shanya",
+    "rbawden",
+    "BigScienceBiasEval",
+    "gsarti",
+    "Helsinki-NLP",
+    "Muennighoff",
+    "facebook",
+    "codeparrot",
+}
 
 # These are the metrics with which templates can be tagged
 METRICS = {
diff --git a/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
new file mode 100644
index 000000000..0d044c303
--- /dev/null
+++ b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
@@ -0,0 +1,45 @@
+dataset: codeparrot/codecomplex
+subset: codeparrot--codecomplex
+templates:
+  5b108b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b108b1c-7514-488f-99ed-3ca5da70e103
+    jinja: '{{ code }}
+      What is the complexity of the previous code?
+      |||
+      {{ complexity }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: what
+    reference: ''
+  1d85c898-70fe-4a51-be37-5111be357762: !Template
+    answer_choices: null
+    id: 1d85c898-70fe-4a51-be37-5111be357762
+    jinja: "Identify the complexity of the following code as constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard. {{ code }} |||{{ complexity }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: false
+    name: identify
+    reference: ''
+  5d85c898-70fe-4a51-be37-5111be357762: !Template
+    answer_choices: null
+    id: 5d85c898-70fe-4a51-be37-5111be357762
+    jinja: "{{ code }} Which one is the correct complexity of the code snippet: constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard? |||{{ complexity }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: false
+    name: identify
+    reference: ''

From 0b99615616b80932b8a2bd350e38ab27a438e077 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Fri, 22 Jul 2022 19:38:53 +0200
Subject: [PATCH 02/36] Adapt names

---
 .../codecomplex/codeparrot--codecomplex/templates.yaml      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
index 0d044c303..5eed0aed5 100644
--- a/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
+++ b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
@@ -15,7 +15,7 @@ templates:
       metrics:
       - Other
       original_task: true
-    name: what
+    name: whatcomplexity
     reference: ''
   1d85c898-70fe-4a51-be37-5111be357762: !Template
     answer_choices: null
@@ -28,7 +28,7 @@ templates:
       metrics:
       - Other
       original_task: false
-    name: identify
+    name: identifycomplexity
     reference: ''
   5d85c898-70fe-4a51-be37-5111be357762: !Template
     answer_choices: null
@@ -41,5 +41,5 @@ templates:
       metrics:
       - Other
       original_task: false
-    name: identify
+    name: whichcomplexity
     reference: ''

From ae7f431efab7f5ef35afa7ab94a53186684e07f4 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 28 Jul 2022 15:46:44 +0200
Subject: [PATCH 03/36] complexity -> time complexity

---
 .../codecomplex/codeparrot--codecomplex/templates.yaml      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
index 5eed0aed5..c3280d715 100644
--- a/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
+++ b/promptsource/templates/codeparrot/codecomplex/codeparrot--codecomplex/templates.yaml
@@ -5,7 +5,7 @@ templates:
     answer_choices: null
     id: 5b108b1c-7514-488f-99ed-3ca5da70e103
     jinja: '{{ code }}
-      What is the complexity of the previous code?
+      What is the time complexity of the previous code?
       |||
       {{ complexity }}'
     metadata: !TemplateMetadata
@@ -20,7 +20,7 @@ templates:
   1d85c898-70fe-4a51-be37-5111be357762: !Template
     answer_choices: null
     id: 1d85c898-70fe-4a51-be37-5111be357762
-    jinja: "Identify the complexity of the following code as constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard. {{ code }} |||{{ complexity }}"
+    jinja: "Identify the time complexity of the following code as constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard. {{ code }} Complexity: |||{{ complexity }}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       languages:
@@ -33,7 +33,7 @@ templates:
   5d85c898-70fe-4a51-be37-5111be357762: !Template
     answer_choices: null
     id: 5d85c898-70fe-4a51-be37-5111be357762
-    jinja: "{{ code }} Which one is the correct complexity of the code snippet: constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard? |||{{ complexity }}"
+    jinja: "{{ code }} Which one is the correct time complexity of the code snippet: constant, linear, quadratic, cubic, log(n), nlog(n) or NP-hard? |||{{ complexity }}"
     metadata: !TemplateMetadata
       choices_in_prompt: false
       languages:

From 27f6a43a203f23c33337ecc2362e9834085700e3 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Sat, 30 Jul 2022 10:34:31 +0200
Subject: [PATCH 04/36] Add clue @07e1c20b2f4e8ac1af98d9a7a9cf3d05f007f36d

---
 .../templates/clue/afqmc/templates.yaml       | 69 ++++++++++++++
 promptsource/templates/clue/c3/templates.yaml | 82 +++++++++++++++++
 .../templates/clue/cluewsc2020/templates.yaml | 78 ++++++++++++++++
 .../templates/clue/cmrc2018/templates.yaml    | 76 ++++++++++++++++
 .../templates/clue/csl/templates.yaml         | 90 +++++++++++++++++++
 .../templates/clue/drcd/templates.yaml        | 76 ++++++++++++++++
 .../templates/clue/tnews/templates.yaml       | 62 +++++++++++++
 7 files changed, 533 insertions(+)
 create mode 100644 promptsource/templates/clue/afqmc/templates.yaml
 create mode 100644 promptsource/templates/clue/c3/templates.yaml
 create mode 100644 promptsource/templates/clue/cluewsc2020/templates.yaml
 create mode 100644 promptsource/templates/clue/cmrc2018/templates.yaml
 create mode 100644 promptsource/templates/clue/csl/templates.yaml
 create mode 100644 promptsource/templates/clue/drcd/templates.yaml
 create mode 100644 promptsource/templates/clue/tnews/templates.yaml

diff --git a/promptsource/templates/clue/afqmc/templates.yaml b/promptsource/templates/clue/afqmc/templates.yaml
new file mode 100644
index 000000000..6498849d5
--- /dev/null
+++ b/promptsource/templates/clue/afqmc/templates.yaml
@@ -0,0 +1,69 @@
+dataset: clue
+subset: afqmc
+templates:
+  997437fd-6888-482d-95e9-ffd867b497ee: !Template
+    answer_choices: no ||| yes
+    id: 997437fd-6888-482d-95e9-ffd867b497ee
+    jinja: 'Do "{{ sentence1 }}" and "{{ sentence2 }}" express the same thing?
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: express_same_yes_no
+    reference: ''
+  a28370c0-d43b-405c-a9b1-4d77b3a27244: !Template
+    answer_choices: no ||| yes
+    id: a28370c0-d43b-405c-a9b1-4d77b3a27244
+    jinja: "\"{{ sentence1 }}\" and \"{{ sentence2 }}\" have the same meaning. Would\
+      \ you agree? Answer yes or no. \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: same_meaning_agree
+    reference: ''
+  d8c303a6-61a4-47f9-8623-cc72cc3294eb: !Template
+    answer_choices: null
+    id: d8c303a6-61a4-47f9-8623-cc72cc3294eb
+    jinja: 'Generate another sentence that has the same meaning as "{{ sentence1 }}".
+
+      |||
+
+      {% if label == 1 %}
+
+      {{ sentence2}}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_similar_sentence
+    reference: ''
+  e3fcaefd-4e8e-4491-aab7-8efeb67a2909: !Template
+    answer_choices: no ||| yes
+    id: e3fcaefd-4e8e-4491-aab7-8efeb67a2909
+    jinja: "Sentence 1: {{ sentence1 }}\nSentence 2: {{ sentence2 }}\nAre the two\
+      \ sentences similar? Yes or no? \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: is_similar_yes_no
+    reference: ''
diff --git a/promptsource/templates/clue/c3/templates.yaml b/promptsource/templates/clue/c3/templates.yaml
new file mode 100644
index 000000000..7856a1a1f
--- /dev/null
+++ b/promptsource/templates/clue/c3/templates.yaml
@@ -0,0 +1,82 @@
+dataset: clue
+subset: c3
+templates:
+  51b3c3fe-2fa2-474a-81f9-5b421c884109: !Template
+    answer_choices: '{{ choice | join(" ||| ") }}'
+    id: 51b3c3fe-2fa2-474a-81f9-5b421c884109
+    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
+      \ the dialogue / passage above, use the following options to answer the question\
+      \ \"{{question}}\".\nOptions: \n- {{ answer_choices | join('\\n- ') }}\n|||\n\
+      {{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: answer-question-affirmative
+    reference: ''
+  5e06f05f-d7dd-4329-b6d8-3a62dcdba838: !Template
+    answer_choices: '{{ choice | join(" ||| ") }}'
+    id: 5e06f05f-d7dd-4329-b6d8-3a62dcdba838
+    jinja: "Passage: {% for statement in context %} \n{{ statement }}\n{% endfor %}\n\
+      Question: \"{{question}}\"\nAnswer choices: {{ answer_choices[:-1] | join(',\
+      \ ') }}, or  {{ answer_choices[-1] }}?\n|||\n{{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: question_choices_context
+    reference: ''
+  63b5e5df-40d3-47ee-b77e-bf385c042fa9: !Template
+    answer_choices: null
+    id: 63b5e5df-40d3-47ee-b77e-bf385c042fa9
+    jinja: "Passage: {% for statement in context %} \n{{ statement }}\n{% endfor %}\n\
+      What kind of question would elicit an answer response of {{ answer }}?\n|||\n\
+      {{ question }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_question
+    reference: ''
+  a5820d05-a8df-4e31-a284-6969e478174b: !Template
+    answer_choices: '{{ choice | join('' ||| '') }}'
+    id: a5820d05-a8df-4e31-a284-6969e478174b
+    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
+      \ the dialogue / passage above, what is the answer for the question \"{{question}}\"\
+      \nAnswer choices: {{ answer_choices[:-1] | join(', ') }}, or  {{ answer_choices[-1]\
+      \ }}?\n|||\n{{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: answer-question-interrogative
+    reference: ''
+  f15acc3f-e067-488f-b426-f65aa604da55: !Template
+    answer_choices: null
+    id: f15acc3f-e067-488f-b426-f65aa604da55
+    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
+      \ the dialogue / passage above, what is the answer for the question \"{{question}}\"\
+      \n|||\n{{ answer }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      - BLEU
+      - Other
+      original_task: false
+    name: answer-question-interrogative-no-choices
+    reference: ''
diff --git a/promptsource/templates/clue/cluewsc2020/templates.yaml b/promptsource/templates/clue/cluewsc2020/templates.yaml
new file mode 100644
index 000000000..4b22f666c
--- /dev/null
+++ b/promptsource/templates/clue/cluewsc2020/templates.yaml
@@ -0,0 +1,78 @@
+dataset: clue
+subset: cluewsc2020
+templates:
+  321f55bb-c725-4fbf-bb7e-d46ea2f510b8: !Template
+    answer_choices: correct ||| wrong
+    id: 321f55bb-c725-4fbf-bb7e-d46ea2f510b8
+    jinja: 'In the class, a teacher asks what  the word "{{ target[''span2_text'']
+      }}" refers to in the text of "{{ text }}". The student answers "{{ target[''span1_text'']
+      }}". The teacher would say what? {{ answer_choices[0] | capitalize }} or {{answer_choices[1]
+      }}?
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: teacher_asking_student
+    reference: ''
+  7282b4b5-f854-42af-8e75-d509608d97bb: !Template
+    answer_choices: null
+    id: 7282b4b5-f854-42af-8e75-d509608d97bb
+    jinja: 'What does the word "{{ target[''span2_text''] }}" refers to in the text
+      of "{{ text }}"?
+
+      |||
+
+      {% if label == 0 %}
+
+      {{ target[''span1_text''] }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_correct_response
+    reference: ''
+  e649a609-f7b2-43da-800d-a32090e92221: !Template
+    answer_choices: yes ||| no
+    id: e649a609-f7b2-43da-800d-a32090e92221
+    jinja: "In the sentence \"{{ text }}\", does \"{{ target['span2_text'] }}\" refer\
+      \ to \"{{ target['span1_text'] }}\"? \n|||\n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: are_they_same
+    reference: ''
+  fc436a38-d9f5-4d17-bcf8-1e506bba5681: !Template
+    answer_choices: yes ||| no
+    id: fc436a38-d9f5-4d17-bcf8-1e506bba5681
+    jinja: 'In the sentence "{{ text }}", the word "{{ target[''span2_text''] }}"
+      refers to "{{ target[''span1_text''] }}". Answer {{ answer_choices[0] }} if
+      you agree; otherwise, answer {{ answer_choices[1] }}.
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: affirmative_are_they_same
+    reference: ''
diff --git a/promptsource/templates/clue/cmrc2018/templates.yaml b/promptsource/templates/clue/cmrc2018/templates.yaml
new file mode 100644
index 000000000..5d2d6b2d5
--- /dev/null
+++ b/promptsource/templates/clue/cmrc2018/templates.yaml
@@ -0,0 +1,76 @@
+dataset: clue
+subset: cmrc2018
+templates:
+  3bba02e6-9266-418b-9ba0-4f71755cf3b6: !Template
+    answer_choices: null
+    id: 3bba02e6-9266-418b-9ba0-4f71755cf3b6
+    jinja: 'Given this context "{{ context }}", generate a question that would return
+      the answer of "{{ answers[''text''][0] }}".
+
+      |||
+
+      {{ question }} '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_question
+    reference: ''
+  8fe02215-7881-4a61-a6e7-579680e40b9b: !Template
+    answer_choices: null
+    id: 8fe02215-7881-4a61-a6e7-579680e40b9b
+    jinja: "In an exam, you are asked {{ question }}, and you are tasked to find the\
+      \ answer from the following passage. \n{{ context }}\nWhat's the answer?\n|||\n\
+      {{ answers['text'][0] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: in_an_exam
+    reference: ''
+  9e82f5da-b206-4758-94e6-085cf2608378: !Template
+    answer_choices: null
+    id: 9e82f5da-b206-4758-94e6-085cf2608378
+    jinja: '{{ context }}
+
+      The answer to {{ question }} is in the passage above. What is it?
+
+      |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_in_the_passage
+    reference: ''
+  9fb15385-814e-419a-b862-2d4e06a58ef6: !Template
+    answer_choices: null
+    id: 9fb15385-814e-419a-b862-2d4e06a58ef6
+    jinja: 'Answer the question using the given context.
+
+      Question: {{ question }}
+
+      Context: {{ context }}
+
+      Answer: |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_following_question
+    reference: ''
diff --git a/promptsource/templates/clue/csl/templates.yaml b/promptsource/templates/clue/csl/templates.yaml
new file mode 100644
index 000000000..2ba03f7da
--- /dev/null
+++ b/promptsource/templates/clue/csl/templates.yaml
@@ -0,0 +1,90 @@
+dataset: clue
+subset: csl
+templates:
+  219679f8-a02f-4ee3-91c7-9ed4726dd828: !Template
+    answer_choices: no ||| yes
+    id: 219679f8-a02f-4ee3-91c7-9ed4726dd828
+    jinja: 'After John wrote the abstract "{{abst}}", he wrote these keywords "{{
+      keyword | join('', '') }}". Do you think his choice of keywords was correct?
+      Answer {{ answer_choices[1]}} or {{ answer_choices[0]}}.
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: write_keywords_after_abstract
+    reference: ''
+  2e851dd2-2677-415a-ad90-5d885aa91fdc: !Template
+    answer_choices: no ||| yes
+    id: 2e851dd2-2677-415a-ad90-5d885aa91fdc
+    jinja: 'Do these keywords "{{ keyword | join('', '') }}" represent key concepts
+      in the abstract "{{ abst }}"?
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: do_represent
+    reference: ''
+  2fa6151b-a296-4cd5-83ca-dcd434b831be: !Template
+    answer_choices: null
+    id: 2fa6151b-a296-4cd5-83ca-dcd434b831be
+    jinja: 'Given the abstract {{abst}}, list out {{ keyword | length }} keywords
+      for it.
+
+      |||
+
+      {% if label == 1 %}
+
+      {{ keyword | join('', '') }}
+
+      {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_keywords
+    reference: ''
+  aaf47f6f-fd8f-4180-8d85-e4c7df088ac6: !Template
+    answer_choices: no ||| yes
+    id: aaf47f6f-fd8f-4180-8d85-e4c7df088ac6
+    jinja: 'A scholar used "{{ keyword | join('', '') }}" as search terms. Do you
+      think the search engine would return the abstract "{{abst}}"? Answer {{ answer_choices[1]
+      }} or {{ answer_choices[0] }}.
+
+      |||
+
+      {{ answer_choices[label] }}
+
+
+      '
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: search_terms
+    reference: ''
diff --git a/promptsource/templates/clue/drcd/templates.yaml b/promptsource/templates/clue/drcd/templates.yaml
new file mode 100644
index 000000000..87d58e134
--- /dev/null
+++ b/promptsource/templates/clue/drcd/templates.yaml
@@ -0,0 +1,76 @@
+dataset: clue
+subset: drcd
+templates:
+  2b2454d1-4375-4fb3-93a5-8c1e4ee605ea: !Template
+    answer_choices: null
+    id: 2b2454d1-4375-4fb3-93a5-8c1e4ee605ea
+    jinja: 'Answer the question using the given context.
+
+      Question: {{ question }}
+
+      Context: {{ context }}
+
+      Answer: |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_following_question
+    reference: ''
+  41aebf75-a867-455b-a5dc-519ab83cf24f: !Template
+    answer_choices: null
+    id: 41aebf75-a867-455b-a5dc-519ab83cf24f
+    jinja: '{{ context }}
+
+      The answer to {{ question }} is in the passage above. What is it?
+
+      |||
+
+      {{ answers[''text''][0] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: answer_in_the_passage
+    reference: ''
+  ac20087c-80a0-4965-8cab-d8cb6f90a555: !Template
+    answer_choices: null
+    id: ac20087c-80a0-4965-8cab-d8cb6f90a555
+    jinja: 'Given this context "{{ context }}", generate a question that would return
+      the answer of "{{ answers[''text''][0] }}".
+
+      |||
+
+      {{ question }} '
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - ROUGE
+      original_task: false
+    name: generate_question
+    reference: ''
+  b2684f23-b191-4e6d-9dc5-12b1d7d4cf49: !Template
+    answer_choices: null
+    id: b2684f23-b191-4e6d-9dc5-12b1d7d4cf49
+    jinja: "In an exam, you are asked {{ question }}, and you are tasked to find the\
+      \ answer from the following passage. \n{{ context }}\nWhat's the answer?\n|||\n\
+      {{ answers['text'][0] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - zh
+      metrics:
+      - Squad
+      original_task: true
+    name: in_an_exam
+    reference: ''
diff --git a/promptsource/templates/clue/tnews/templates.yaml b/promptsource/templates/clue/tnews/templates.yaml
new file mode 100644
index 000000000..685170cc5
--- /dev/null
+++ b/promptsource/templates/clue/tnews/templates.yaml
@@ -0,0 +1,62 @@
+dataset: clue
+subset: tnews
+templates:
+  0c965fcd-d5e9-4e6a-b8ec-13253ed7bf4a: !Template
+    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
+      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
+      news ||| stock ||| agriculture ||| game
+    id: 0c965fcd-d5e9-4e6a-b8ec-13253ed7bf4a
+    jinja: 'Classify the title "{{ sentence }}" into the following topics:
+
+      - {{ answer_choices | join(''\n- '') }}
+
+      Topic: |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: classify_title
+    reference: ''
+  26e75138-7aa4-4b13-b2fa-7cd4ad5484b3: !Template
+    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
+      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
+      news ||| stock ||| agriculture ||| game
+    id: 26e75138-7aa4-4b13-b2fa-7cd4ad5484b3
+    jinja: "Given the topics of {{answer_choices[:-1] | join(', ') }}, and {{ answer_choices[-1]\
+      \ }}, specify which of them best represents the following sentence:\n{{ sentence\
+      \ }}\n\nBest: ||| \n{{ answer_choices[label] }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: best_represent
+    reference: ''
+  c242254c-bf5d-4efb-9dc3-51717bab7f78: !Template
+    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
+      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
+      news ||| stock ||| agriculture ||| game
+    id: c242254c-bf5d-4efb-9dc3-51717bab7f78
+    jinja: 'What topic does the following news title "{{ sentence }}" belong to? {{
+      answer_choices[0] | capitalize }}, {{ answer_choices[1:-1] | join('', '') }},
+      or {{ answer_choices[-1] }}?
+
+      |||
+
+      {{ answer_choices[label] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - zh
+      metrics:
+      - Accuracy
+      original_task: true
+    name: what_title
+    reference: ''

From cd718e0ff56c4f46063bae80a1a3f9c918581073 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 15:21:22 +0200
Subject: [PATCH 05/36] Add file

---
 .../templates.yaml                            | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)
 create mode 100644 promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml

diff --git a/promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml b/promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml
new file mode 100644
index 000000000..d0bfd8e6c
--- /dev/null
+++ b/promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml
@@ -0,0 +1,77 @@
+dataset: codeparrot/github-jupyter-text-code-pairs
+subset: github-jupyter-text-code-pairs
+templates:
+  5b218b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b1c-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ markdown }}"
+
+      Please write code following the instructions.
+      
+      |||
+
+      {{ code }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: code
+    reference: ''
+  5b218b2c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2c-7514-488f-99ed-3ca5da70e103
+    jinja: 'I am working on the file "{{ path }}".
+
+      The first task is:
+
+      {{ markdown }}
+
+      Can you write the code for it?
+      
+      |||
+
+      {{ code }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: taskcode
+    reference: ''
+  4d85c898-70fe-4a51-be37-5111be357762: !Template
+    answer_choices: null
+    id: 4d85c898-70fe-4a51-be37-5111be357762
+    jinja: "{{ markdown }}\n|||{{ code }}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: false
+    name: markdowncode
+    reference: ''
+  8d85c898-70fe-4a51-be37-5111be357762: !Template
+    answer_choices: null
+    id: 8d85c898-70fe-4a51-be37-5111be357762
+    jinja: '{{ code }}
+    
+    Given the above code, generate some markdown instructions for it.
+
+    |||
+    
+    {{ markdown }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: false
+    name: genmarkdown
+    reference: ''

From 577d6e6b277ac9874b65bc6a0263117795fe1782 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 15:44:10 +0200
Subject: [PATCH 06/36] Add code

---
 promptsource/templates.py                     |  1 +
 .../python-state-changes/templates.yaml       | 87 +++++++++++++++++++
 .../templates/codeparrot/apps/templates.yaml  | 65 ++++++++++++++
 .../templates.yaml                            |  6 +-
 4 files changed, 156 insertions(+), 3 deletions(-)
 create mode 100644 promptsource/templates/Fraser/python-state-changes/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/apps/templates.yaml
 rename promptsource/templates/codeparrot/{codecomplex => }/github-jupyter-text-code-pairs/templates.yaml (92%)

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 943f8611e..484d78107 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -41,6 +41,7 @@
     "Muennighoff",
     "facebook",
     "codeparrot",
+    "Fraser",
 }
 
 # These are the metrics with which templates can be tagged
diff --git a/promptsource/templates/Fraser/python-state-changes/templates.yaml b/promptsource/templates/Fraser/python-state-changes/templates.yaml
new file mode 100644
index 000000000..70af6a1bd
--- /dev/null
+++ b/promptsource/templates/Fraser/python-state-changes/templates.yaml
@@ -0,0 +1,87 @@
+dataset: Fraser/python-state-changes
+subset:
+templates:
+  2b358b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 2b358b1c-7514-488f-99ed-3ca5da70e103
+    jinja: 'Starting variables:
+    
+      {{ start }}
+
+      Applied code:
+
+      {{code}}
+
+      Ending variables:
+      
+      |||
+      {{ end }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: startend
+    reference: ''
+  1b218b2c-8514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 1b218b2c-8514-488f-99ed-3ca5da70e103
+    jinja: 'I applied "{{code}}" given "{{ start }}".
+
+      What are the new values of the variables now?
+
+      |||
+
+      {{ end }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: newval
+    reference: ''
+  5f318b2c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f318b2c-7514-488f-99ed-3ca5da70e103
+    jinja: 'The final variables are:
+    
+      {{ end }}
+
+      We know that the code "{{code}}" was applied.
+
+      What were the variables at the beginning?
+
+      |||
+      {{ start }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: varbeg
+    reference: ''
+  5b918b2c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b918b2c-7514-488f-99ed-3ca5da70e103
+    jinja: 'What code do I need to apply to get from start to end?
+    
+      Start: {{ start }}
+
+      End: {{ end }}
+
+      Needed code: ||| {{ code }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: needcode
+    reference: ''
diff --git a/promptsource/templates/codeparrot/apps/templates.yaml b/promptsource/templates/codeparrot/apps/templates.yaml
new file mode 100644
index 000000000..d90cc4a77
--- /dev/null
+++ b/promptsource/templates/codeparrot/apps/templates.yaml
@@ -0,0 +1,65 @@
+dataset: codeparrot/apps
+subset: all
+templates:
+  5b318b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b318b1c-7514-488f-99ed-3ca5da70e103
+    jinja: 'Solve in Python:
+    
+      {{ question }}
+      
+      |||
+
+      {{ solution }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: qsol
+    reference: ''
+  5b218b2c-8514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2c-8514-488f-99ed-3ca5da70e103
+    jinja: '{{ question }}
+      
+
+      Can you solve the above problem using Python?
+
+      |||
+
+      {{ solution }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovesol
+    reference: ''
+  5b318b2c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b318b2c-7514-488f-99ed-3ca5da70e103
+    jinja: 'I found an interesting problem on {{url}}:
+    
+      {{ question }}
+
+      
+      I tried it in Python, but could not do it. Can you solve it?
+      
+
+      |||
+
+      {{ solution }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovesol
+    reference: ''
diff --git a/promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
similarity index 92%
rename from promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml
rename to promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
index d0bfd8e6c..7ac5746f0 100644
--- a/promptsource/templates/codeparrot/codecomplex/github-jupyter-text-code-pairs/templates.yaml
+++ b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
@@ -1,12 +1,12 @@
 dataset: codeparrot/github-jupyter-text-code-pairs
-subset: github-jupyter-text-code-pairs
+subset:
 templates:
   5b218b1c-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
     id: 5b218b1c-7514-488f-99ed-3ca5da70e103
     jinja: '"{{ markdown }}"
 
-      Please write code following the instructions.
+      Please write code following the instructions in jupyter notebook style.
       
       |||
 
@@ -29,7 +29,7 @@ templates:
 
       {{ markdown }}
 
-      Can you write the code for it?
+      Can you write Python code for it?
       
       |||
 

From 4dd78705b08522c9af6ca11635b9139a66ff62cd Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 15:45:18 +0200
Subject: [PATCH 07/36] Rmv unrelated

---
 .../templates/clue/afqmc/templates.yaml       | 69 --------------
 promptsource/templates/clue/c3/templates.yaml | 82 -----------------
 .../templates/clue/cluewsc2020/templates.yaml | 78 ----------------
 .../templates/clue/cmrc2018/templates.yaml    | 76 ----------------
 .../templates/clue/csl/templates.yaml         | 90 -------------------
 .../templates/clue/drcd/templates.yaml        | 76 ----------------
 .../templates/clue/tnews/templates.yaml       | 62 -------------
 7 files changed, 533 deletions(-)
 delete mode 100644 promptsource/templates/clue/afqmc/templates.yaml
 delete mode 100644 promptsource/templates/clue/c3/templates.yaml
 delete mode 100644 promptsource/templates/clue/cluewsc2020/templates.yaml
 delete mode 100644 promptsource/templates/clue/cmrc2018/templates.yaml
 delete mode 100644 promptsource/templates/clue/csl/templates.yaml
 delete mode 100644 promptsource/templates/clue/drcd/templates.yaml
 delete mode 100644 promptsource/templates/clue/tnews/templates.yaml

diff --git a/promptsource/templates/clue/afqmc/templates.yaml b/promptsource/templates/clue/afqmc/templates.yaml
deleted file mode 100644
index 6498849d5..000000000
--- a/promptsource/templates/clue/afqmc/templates.yaml
+++ /dev/null
@@ -1,69 +0,0 @@
-dataset: clue
-subset: afqmc
-templates:
-  997437fd-6888-482d-95e9-ffd867b497ee: !Template
-    answer_choices: no ||| yes
-    id: 997437fd-6888-482d-95e9-ffd867b497ee
-    jinja: 'Do "{{ sentence1 }}" and "{{ sentence2 }}" express the same thing?
-
-      |||
-
-      {{ answer_choices[label] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: express_same_yes_no
-    reference: ''
-  a28370c0-d43b-405c-a9b1-4d77b3a27244: !Template
-    answer_choices: no ||| yes
-    id: a28370c0-d43b-405c-a9b1-4d77b3a27244
-    jinja: "\"{{ sentence1 }}\" and \"{{ sentence2 }}\" have the same meaning. Would\
-      \ you agree? Answer yes or no. \n|||\n{{ answer_choices[label] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: same_meaning_agree
-    reference: ''
-  d8c303a6-61a4-47f9-8623-cc72cc3294eb: !Template
-    answer_choices: null
-    id: d8c303a6-61a4-47f9-8623-cc72cc3294eb
-    jinja: 'Generate another sentence that has the same meaning as "{{ sentence1 }}".
-
-      |||
-
-      {% if label == 1 %}
-
-      {{ sentence2}}
-
-      {% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_similar_sentence
-    reference: ''
-  e3fcaefd-4e8e-4491-aab7-8efeb67a2909: !Template
-    answer_choices: no ||| yes
-    id: e3fcaefd-4e8e-4491-aab7-8efeb67a2909
-    jinja: "Sentence 1: {{ sentence1 }}\nSentence 2: {{ sentence2 }}\nAre the two\
-      \ sentences similar? Yes or no? \n|||\n{{ answer_choices[label] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: is_similar_yes_no
-    reference: ''
diff --git a/promptsource/templates/clue/c3/templates.yaml b/promptsource/templates/clue/c3/templates.yaml
deleted file mode 100644
index 7856a1a1f..000000000
--- a/promptsource/templates/clue/c3/templates.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-dataset: clue
-subset: c3
-templates:
-  51b3c3fe-2fa2-474a-81f9-5b421c884109: !Template
-    answer_choices: '{{ choice | join(" ||| ") }}'
-    id: 51b3c3fe-2fa2-474a-81f9-5b421c884109
-    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
-      \ the dialogue / passage above, use the following options to answer the question\
-      \ \"{{question}}\".\nOptions: \n- {{ answer_choices | join('\\n- ') }}\n|||\n\
-      {{ answer }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: answer-question-affirmative
-    reference: ''
-  5e06f05f-d7dd-4329-b6d8-3a62dcdba838: !Template
-    answer_choices: '{{ choice | join(" ||| ") }}'
-    id: 5e06f05f-d7dd-4329-b6d8-3a62dcdba838
-    jinja: "Passage: {% for statement in context %} \n{{ statement }}\n{% endfor %}\n\
-      Question: \"{{question}}\"\nAnswer choices: {{ answer_choices[:-1] | join(',\
-      \ ') }}, or  {{ answer_choices[-1] }}?\n|||\n{{ answer }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: question_choices_context
-    reference: ''
-  63b5e5df-40d3-47ee-b77e-bf385c042fa9: !Template
-    answer_choices: null
-    id: 63b5e5df-40d3-47ee-b77e-bf385c042fa9
-    jinja: "Passage: {% for statement in context %} \n{{ statement }}\n{% endfor %}\n\
-      What kind of question would elicit an answer response of {{ answer }}?\n|||\n\
-      {{ question }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_question
-    reference: ''
-  a5820d05-a8df-4e31-a284-6969e478174b: !Template
-    answer_choices: '{{ choice | join('' ||| '') }}'
-    id: a5820d05-a8df-4e31-a284-6969e478174b
-    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
-      \ the dialogue / passage above, what is the answer for the question \"{{question}}\"\
-      \nAnswer choices: {{ answer_choices[:-1] | join(', ') }}, or  {{ answer_choices[-1]\
-      \ }}?\n|||\n{{ answer }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: answer-question-interrogative
-    reference: ''
-  f15acc3f-e067-488f-b426-f65aa604da55: !Template
-    answer_choices: null
-    id: f15acc3f-e067-488f-b426-f65aa604da55
-    jinja: "{% for statement in context %} \n{{ statement }}\n{% endfor %}\nGiven\
-      \ the dialogue / passage above, what is the answer for the question \"{{question}}\"\
-      \n|||\n{{ answer }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      - BLEU
-      - Other
-      original_task: false
-    name: answer-question-interrogative-no-choices
-    reference: ''
diff --git a/promptsource/templates/clue/cluewsc2020/templates.yaml b/promptsource/templates/clue/cluewsc2020/templates.yaml
deleted file mode 100644
index 4b22f666c..000000000
--- a/promptsource/templates/clue/cluewsc2020/templates.yaml
+++ /dev/null
@@ -1,78 +0,0 @@
-dataset: clue
-subset: cluewsc2020
-templates:
-  321f55bb-c725-4fbf-bb7e-d46ea2f510b8: !Template
-    answer_choices: correct ||| wrong
-    id: 321f55bb-c725-4fbf-bb7e-d46ea2f510b8
-    jinja: 'In the class, a teacher asks what  the word "{{ target[''span2_text'']
-      }}" refers to in the text of "{{ text }}". The student answers "{{ target[''span1_text'']
-      }}". The teacher would say what? {{ answer_choices[0] | capitalize }} or {{answer_choices[1]
-      }}?
-
-      |||
-
-      {{ answer_choices[label] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: teacher_asking_student
-    reference: ''
-  7282b4b5-f854-42af-8e75-d509608d97bb: !Template
-    answer_choices: null
-    id: 7282b4b5-f854-42af-8e75-d509608d97bb
-    jinja: 'What does the word "{{ target[''span2_text''] }}" refers to in the text
-      of "{{ text }}"?
-
-      |||
-
-      {% if label == 0 %}
-
-      {{ target[''span1_text''] }}
-
-      {% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_correct_response
-    reference: ''
-  e649a609-f7b2-43da-800d-a32090e92221: !Template
-    answer_choices: yes ||| no
-    id: e649a609-f7b2-43da-800d-a32090e92221
-    jinja: "In the sentence \"{{ text }}\", does \"{{ target['span2_text'] }}\" refer\
-      \ to \"{{ target['span1_text'] }}\"? \n|||\n{{ answer_choices[label] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: are_they_same
-    reference: ''
-  fc436a38-d9f5-4d17-bcf8-1e506bba5681: !Template
-    answer_choices: yes ||| no
-    id: fc436a38-d9f5-4d17-bcf8-1e506bba5681
-    jinja: 'In the sentence "{{ text }}", the word "{{ target[''span2_text''] }}"
-      refers to "{{ target[''span1_text''] }}". Answer {{ answer_choices[0] }} if
-      you agree; otherwise, answer {{ answer_choices[1] }}.
-
-      |||
-
-      {{ answer_choices[label] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: affirmative_are_they_same
-    reference: ''
diff --git a/promptsource/templates/clue/cmrc2018/templates.yaml b/promptsource/templates/clue/cmrc2018/templates.yaml
deleted file mode 100644
index 5d2d6b2d5..000000000
--- a/promptsource/templates/clue/cmrc2018/templates.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-dataset: clue
-subset: cmrc2018
-templates:
-  3bba02e6-9266-418b-9ba0-4f71755cf3b6: !Template
-    answer_choices: null
-    id: 3bba02e6-9266-418b-9ba0-4f71755cf3b6
-    jinja: 'Given this context "{{ context }}", generate a question that would return
-      the answer of "{{ answers[''text''][0] }}".
-
-      |||
-
-      {{ question }} '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_question
-    reference: ''
-  8fe02215-7881-4a61-a6e7-579680e40b9b: !Template
-    answer_choices: null
-    id: 8fe02215-7881-4a61-a6e7-579680e40b9b
-    jinja: "In an exam, you are asked {{ question }}, and you are tasked to find the\
-      \ answer from the following passage. \n{{ context }}\nWhat's the answer?\n|||\n\
-      {{ answers['text'][0] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: in_an_exam
-    reference: ''
-  9e82f5da-b206-4758-94e6-085cf2608378: !Template
-    answer_choices: null
-    id: 9e82f5da-b206-4758-94e6-085cf2608378
-    jinja: '{{ context }}
-
-      The answer to {{ question }} is in the passage above. What is it?
-
-      |||
-
-      {{ answers[''text''][0] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: answer_in_the_passage
-    reference: ''
-  9fb15385-814e-419a-b862-2d4e06a58ef6: !Template
-    answer_choices: null
-    id: 9fb15385-814e-419a-b862-2d4e06a58ef6
-    jinja: 'Answer the question using the given context.
-
-      Question: {{ question }}
-
-      Context: {{ context }}
-
-      Answer: |||
-
-      {{ answers[''text''][0] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: answer_following_question
-    reference: ''
diff --git a/promptsource/templates/clue/csl/templates.yaml b/promptsource/templates/clue/csl/templates.yaml
deleted file mode 100644
index 2ba03f7da..000000000
--- a/promptsource/templates/clue/csl/templates.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-dataset: clue
-subset: csl
-templates:
-  219679f8-a02f-4ee3-91c7-9ed4726dd828: !Template
-    answer_choices: no ||| yes
-    id: 219679f8-a02f-4ee3-91c7-9ed4726dd828
-    jinja: 'After John wrote the abstract "{{abst}}", he wrote these keywords "{{
-      keyword | join('', '') }}". Do you think his choice of keywords was correct?
-      Answer {{ answer_choices[1]}} or {{ answer_choices[0]}}.
-
-      |||
-
-      {{ answer_choices[label] }}
-
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: write_keywords_after_abstract
-    reference: ''
-  2e851dd2-2677-415a-ad90-5d885aa91fdc: !Template
-    answer_choices: no ||| yes
-    id: 2e851dd2-2677-415a-ad90-5d885aa91fdc
-    jinja: 'Do these keywords "{{ keyword | join('', '') }}" represent key concepts
-      in the abstract "{{ abst }}"?
-
-      |||
-
-      {{ answer_choices[label] }}
-
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: do_represent
-    reference: ''
-  2fa6151b-a296-4cd5-83ca-dcd434b831be: !Template
-    answer_choices: null
-    id: 2fa6151b-a296-4cd5-83ca-dcd434b831be
-    jinja: 'Given the abstract {{abst}}, list out {{ keyword | length }} keywords
-      for it.
-
-      |||
-
-      {% if label == 1 %}
-
-      {{ keyword | join('', '') }}
-
-      {% endif %}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_keywords
-    reference: ''
-  aaf47f6f-fd8f-4180-8d85-e4c7df088ac6: !Template
-    answer_choices: no ||| yes
-    id: aaf47f6f-fd8f-4180-8d85-e4c7df088ac6
-    jinja: 'A scholar used "{{ keyword | join('', '') }}" as search terms. Do you
-      think the search engine would return the abstract "{{abst}}"? Answer {{ answer_choices[1]
-      }} or {{ answer_choices[0] }}.
-
-      |||
-
-      {{ answer_choices[label] }}
-
-
-      '
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: search_terms
-    reference: ''
diff --git a/promptsource/templates/clue/drcd/templates.yaml b/promptsource/templates/clue/drcd/templates.yaml
deleted file mode 100644
index 87d58e134..000000000
--- a/promptsource/templates/clue/drcd/templates.yaml
+++ /dev/null
@@ -1,76 +0,0 @@
-dataset: clue
-subset: drcd
-templates:
-  2b2454d1-4375-4fb3-93a5-8c1e4ee605ea: !Template
-    answer_choices: null
-    id: 2b2454d1-4375-4fb3-93a5-8c1e4ee605ea
-    jinja: 'Answer the question using the given context.
-
-      Question: {{ question }}
-
-      Context: {{ context }}
-
-      Answer: |||
-
-      {{ answers[''text''][0] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: answer_following_question
-    reference: ''
-  41aebf75-a867-455b-a5dc-519ab83cf24f: !Template
-    answer_choices: null
-    id: 41aebf75-a867-455b-a5dc-519ab83cf24f
-    jinja: '{{ context }}
-
-      The answer to {{ question }} is in the passage above. What is it?
-
-      |||
-
-      {{ answers[''text''][0] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: answer_in_the_passage
-    reference: ''
-  ac20087c-80a0-4965-8cab-d8cb6f90a555: !Template
-    answer_choices: null
-    id: ac20087c-80a0-4965-8cab-d8cb6f90a555
-    jinja: 'Given this context "{{ context }}", generate a question that would return
-      the answer of "{{ answers[''text''][0] }}".
-
-      |||
-
-      {{ question }} '
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - ROUGE
-      original_task: false
-    name: generate_question
-    reference: ''
-  b2684f23-b191-4e6d-9dc5-12b1d7d4cf49: !Template
-    answer_choices: null
-    id: b2684f23-b191-4e6d-9dc5-12b1d7d4cf49
-    jinja: "In an exam, you are asked {{ question }}, and you are tasked to find the\
-      \ answer from the following passage. \n{{ context }}\nWhat's the answer?\n|||\n\
-      {{ answers['text'][0] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: false
-      languages:
-      - zh
-      metrics:
-      - Squad
-      original_task: true
-    name: in_an_exam
-    reference: ''
diff --git a/promptsource/templates/clue/tnews/templates.yaml b/promptsource/templates/clue/tnews/templates.yaml
deleted file mode 100644
index 685170cc5..000000000
--- a/promptsource/templates/clue/tnews/templates.yaml
+++ /dev/null
@@ -1,62 +0,0 @@
-dataset: clue
-subset: tnews
-templates:
-  0c965fcd-d5e9-4e6a-b8ec-13253ed7bf4a: !Template
-    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
-      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
-      news ||| stock ||| agriculture ||| game
-    id: 0c965fcd-d5e9-4e6a-b8ec-13253ed7bf4a
-    jinja: 'Classify the title "{{ sentence }}" into the following topics:
-
-      - {{ answer_choices | join(''\n- '') }}
-
-      Topic: |||
-
-      {{ answer_choices[label] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: classify_title
-    reference: ''
-  26e75138-7aa4-4b13-b2fa-7cd4ad5484b3: !Template
-    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
-      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
-      news ||| stock ||| agriculture ||| game
-    id: 26e75138-7aa4-4b13-b2fa-7cd4ad5484b3
-    jinja: "Given the topics of {{answer_choices[:-1] | join(', ') }}, and {{ answer_choices[-1]\
-      \ }}, specify which of them best represents the following sentence:\n{{ sentence\
-      \ }}\n\nBest: ||| \n{{ answer_choices[label] }}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: best_represent
-    reference: ''
-  c242254c-bf5d-4efb-9dc3-51717bab7f78: !Template
-    answer_choices: story ||| culture ||| entertainment ||| sports ||| finance |||
-      real estate ||| car ||| education ||| tech ||| military ||| travel ||| world
-      news ||| stock ||| agriculture ||| game
-    id: c242254c-bf5d-4efb-9dc3-51717bab7f78
-    jinja: 'What topic does the following news title "{{ sentence }}" belong to? {{
-      answer_choices[0] | capitalize }}, {{ answer_choices[1:-1] | join('', '') }},
-      or {{ answer_choices[-1] }}?
-
-      |||
-
-      {{ answer_choices[label] }}'
-    metadata: !TemplateMetadata
-      choices_in_prompt: true
-      languages:
-      - zh
-      metrics:
-      - Accuracy
-      original_task: true
-    name: what_title
-    reference: ''

From 32554215330c57fb0f5f9408720965a2e06d81cb Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 15:46:44 +0200
Subject: [PATCH 08/36] Fixes

---
 .../github-jupyter-text-code-pairs/templates.yaml         | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
index 7ac5746f0..4b867feb8 100644
--- a/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
+++ b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
@@ -1,9 +1,9 @@
 dataset: codeparrot/github-jupyter-text-code-pairs
 subset:
 templates:
-  5b218b1c-7514-488f-99ed-3ca5da70e103: !Template
+  5b718b1c-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
-    id: 5b218b1c-7514-488f-99ed-3ca5da70e103
+    id: 5b718b1c-7514-488f-99ed-3ca5da70e103
     jinja: '"{{ markdown }}"
 
       Please write code following the instructions in jupyter notebook style.
@@ -20,9 +20,9 @@ templates:
       original_task: true
     name: code
     reference: ''
-  5b218b2c-7514-488f-99ed-3ca5da70e103: !Template
+  5b218b2e-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
-    id: 5b218b2c-7514-488f-99ed-3ca5da70e103
+    id: 5b218b2e-7514-488f-99ed-3ca5da70e103
     jinja: 'I am working on the file "{{ path }}".
 
       The first task is:

From 9b2f3a51c942bcd3ceb86e5d7b311599969a3715 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 15:59:06 +0200
Subject: [PATCH 09/36] fix id

---
 promptsource/templates/codeparrot/apps/templates.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/promptsource/templates/codeparrot/apps/templates.yaml b/promptsource/templates/codeparrot/apps/templates.yaml
index d90cc4a77..13ddb9bc1 100644
--- a/promptsource/templates/codeparrot/apps/templates.yaml
+++ b/promptsource/templates/codeparrot/apps/templates.yaml
@@ -20,9 +20,9 @@ templates:
       original_task: true
     name: qsol
     reference: ''
-  5b218b2c-8514-488f-99ed-3ca5da70e103: !Template
+  5b218b3c-8514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
-    id: 5b218b2c-8514-488f-99ed-3ca5da70e103
+    id: 5b218b3c-8514-488f-99ed-3ca5da70e103
     jinja: '{{ question }}
       
 

From 8f5f343d7a15a612bb6b1a0eac31f6660522788e Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 16:06:12 +0200
Subject: [PATCH 10/36] fix subsets

---
 .../Fraser/python-state-changes/{ => default}/templates.yaml    | 2 +-
 promptsource/templates/codeparrot/apps/{ => all}/templates.yaml | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename promptsource/templates/Fraser/python-state-changes/{ => default}/templates.yaml (99%)
 rename promptsource/templates/codeparrot/apps/{ => all}/templates.yaml (100%)

diff --git a/promptsource/templates/Fraser/python-state-changes/templates.yaml b/promptsource/templates/Fraser/python-state-changes/default/templates.yaml
similarity index 99%
rename from promptsource/templates/Fraser/python-state-changes/templates.yaml
rename to promptsource/templates/Fraser/python-state-changes/default/templates.yaml
index 70af6a1bd..176e042fc 100644
--- a/promptsource/templates/Fraser/python-state-changes/templates.yaml
+++ b/promptsource/templates/Fraser/python-state-changes/default/templates.yaml
@@ -1,5 +1,5 @@
 dataset: Fraser/python-state-changes
-subset:
+subset: default
 templates:
   2b358b1c-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
diff --git a/promptsource/templates/codeparrot/apps/templates.yaml b/promptsource/templates/codeparrot/apps/all/templates.yaml
similarity index 100%
rename from promptsource/templates/codeparrot/apps/templates.yaml
rename to promptsource/templates/codeparrot/apps/all/templates.yaml

From 47c4c2f9decf931ce0f27807cafb869f1acd72ed Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 16:27:21 +0200
Subject: [PATCH 11/36] Fix conflicts

---
 promptsource/templates.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 484d78107..714f8a308 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -41,7 +41,10 @@
     "Muennighoff",
     "facebook",
     "codeparrot",
+    "pasinit",
     "Fraser",
+    "allenai"
+}
 }
 
 # These are the metrics with which templates can be tagged

From dbd42f441fda698e8fc58f7f38311080409bfc1b Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 16:27:43 +0200
Subject: [PATCH 12/36] Fix

---
 promptsource/templates.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/promptsource/templates.py b/promptsource/templates.py
index 714f8a308..df543e1e8 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -43,8 +43,7 @@
     "codeparrot",
     "pasinit",
     "Fraser",
-    "allenai"
-}
+    "allenai",
 }
 
 # These are the metrics with which templates can be tagged

From f31ce9b7634d946c895b80937d757124318bafd9 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 16:39:03 +0200
Subject: [PATCH 13/36] Fix

---
 .../github-jupyter-text-code-pairs/templates.yaml               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename promptsource/templates/codeparrot/{ => codeparrot--github-jupyter-text-code-pairs}/github-jupyter-text-code-pairs/templates.yaml (97%)

diff --git a/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml b/promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml
similarity index 97%
rename from promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
rename to promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml
index 4b867feb8..93e92e329 100644
--- a/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
+++ b/promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml
@@ -1,5 +1,5 @@
 dataset: codeparrot/github-jupyter-text-code-pairs
-subset:
+subset: codeparrot--github-jupyter-text-code-pairs
 templates:
   5b718b1c-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null

From a189e4344406ef7c497b1a9feb9002a58fb3567f Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 17:08:06 +0200
Subject: [PATCH 14/36] Fix

---
 .../github-jupyter-text-code-pairs/templates.yaml               | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename promptsource/templates/codeparrot/{codeparrot--github-jupyter-text-code-pairs => }/github-jupyter-text-code-pairs/templates.yaml (97%)

diff --git a/promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
similarity index 97%
rename from promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml
rename to promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
index 93e92e329..4b867feb8 100644
--- a/promptsource/templates/codeparrot/codeparrot--github-jupyter-text-code-pairs/github-jupyter-text-code-pairs/templates.yaml
+++ b/promptsource/templates/codeparrot/github-jupyter-text-code-pairs/templates.yaml
@@ -1,5 +1,5 @@
 dataset: codeparrot/github-jupyter-text-code-pairs
-subset: codeparrot--github-jupyter-text-code-pairs
+subset:
 templates:
   5b718b1c-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null

From 04d78a32fe5fee050821932a5aec06a08b225bb5 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 22:33:33 +0200
Subject: [PATCH 15/36] Add prompts

---
 promptsource/templates.py                     |  1 +
 .../Python-program-level/templates.yaml       | 39 +++++++++++++
 .../teven/code_contests/templates.yaml        | 55 +++++++++++++++++++
 3 files changed, 95 insertions(+)
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/Python-program-level/templates.yaml
 create mode 100644 promptsource/templates/teven/code_contests/templates.yaml

diff --git a/promptsource/templates.py b/promptsource/templates.py
index df543e1e8..fec4e5e4b 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -44,6 +44,7 @@
     "pasinit",
     "Fraser",
     "allenai",
+    "teven",
 }
 
 # These are the metrics with which templates can be tagged
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/Python-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/Python-program-level/templates.yaml
new file mode 100644
index 000000000..8b52d97be
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/Python-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: Python-program-level
+templates:
+  5f718b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f718b1c-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in Python:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solpy
+    reference: ''
+  5b218b2e-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2e-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in Python?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovepy
+    reference: ''
diff --git a/promptsource/templates/teven/code_contests/templates.yaml b/promptsource/templates/teven/code_contests/templates.yaml
new file mode 100644
index 000000000..f1a34ae50
--- /dev/null
+++ b/promptsource/templates/teven/code_contests/templates.yaml
@@ -0,0 +1,55 @@
+dataset: teven/code_contests
+subset:
+templates:
+  5a6946f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
+    id: 5a6946f9-a0e2-4fbb-aee8-b26ead2cf6b8
+    jinja: '{{description}}
+    
+    ||| 
+    
+    {{solution}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: descsol
+    reference: ''
+  1a9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
+    id: 1a9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
+    jinja: "Can you solve the below in {{language}}?
+
+    {{description}}
+    
+    |||
+    
+    {{solution}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: langsol
+    reference: ''
+  9dcb69d1-cad0-4d2f-a7cc-120df233571c: !Template
+    id: 9dcb69d1-cad0-4d2f-a7cc-120df233571c
+    jinja: '{{description}}
+    
+    The above is tricky. Write me a correct solution in {{language}}.
+    
+    |||
+
+    {{solution}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: abovesol
+    reference: ''

From 8a5208f65648a252b72c0aece3d830d208bc02f8 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 22:36:48 +0200
Subject: [PATCH 16/36] Add prompts

---
 promptsource/templates/teven/code_contests/templates.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/promptsource/templates/teven/code_contests/templates.yaml b/promptsource/templates/teven/code_contests/templates.yaml
index f1a34ae50..760759e22 100644
--- a/promptsource/templates/teven/code_contests/templates.yaml
+++ b/promptsource/templates/teven/code_contests/templates.yaml
@@ -3,6 +3,7 @@ subset:
 templates:
   5a6946f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5a6946f9-a0e2-4fbb-aee8-b26ead2cf6b8
+    answer_choices: null
     jinja: '{{description}}
     
     ||| 
@@ -19,6 +20,7 @@ templates:
     reference: ''
   1a9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
     id: 1a9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
+    answer_choices: null
     jinja: "Can you solve the below in {{language}}?
 
     {{description}}
@@ -37,6 +39,7 @@ templates:
     reference: ''
   9dcb69d1-cad0-4d2f-a7cc-120df233571c: !Template
     id: 9dcb69d1-cad0-4d2f-a7cc-120df233571c
+    answer_choices: null
     jinja: '{{description}}
     
     The above is tricky. Write me a correct solution in {{language}}.

From e78317bd755260b762e931eeb379f36a6c5da5b5 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 23:06:44 +0200
Subject: [PATCH 17/36] Add xlcost

---
 .../C++-program-level/templates.yaml          | 39 +++++++++++++++++++
 .../C-program-level/templates.yaml            | 39 +++++++++++++++++++
 .../Csharp-program-level/templates.yaml       | 39 +++++++++++++++++++
 .../Java-program-level/templates.yaml         | 39 +++++++++++++++++++
 .../Javascript-program-level/templates.yaml   | 39 +++++++++++++++++++
 .../PHP-program-level/templates.yaml          | 39 +++++++++++++++++++
 6 files changed, 234 insertions(+)
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/C++-program-level/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/C-program-level/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/Csharp-program-level/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/Javascript-program-level/templates.yaml
 create mode 100644 promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml

diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/C++-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/C++-program-level/templates.yaml
new file mode 100644
index 000000000..8019a6388
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/C++-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: C++-program-level
+templates:
+  5f718b2d-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f718b2d-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in C++:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solcpp
+    reference: ''
+  5b218b2e-7525-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2e-7525-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in C++?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovecpp
+    reference: ''
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/C-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/C-program-level/templates.yaml
new file mode 100644
index 000000000..c7b60b495
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/C-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: C-program-level
+templates:
+  5f718b1c-7514-589f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f718b1c-7514-589f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in C:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solc
+    reference: ''
+  5b218b2e-7514-488f-99ed-4da5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2e-7514-488f-99ed-4da5da70e103
+    jinja: '{{ text }}
+
+      How can the above be solved in C?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovec
+    reference: ''
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/Csharp-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/Csharp-program-level/templates.yaml
new file mode 100644
index 000000000..438e3cc03
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/Csharp-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: Python-program-level
+templates:
+  5f729b1c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f729b1c-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in C#:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solcsharp
+    reference: ''
+  6c218b2e-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 6c218b2e-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in C-Sharp?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovecsharp
+    reference: ''
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
new file mode 100644
index 000000000..10732cd36
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: Python-program-level
+templates:
+  5f729b5c-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f729b5c-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in Java:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solc
+    reference: ''
+  5b218b2e-7579-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b218b2e-7579-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in Java?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovejava
+    reference: ''
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/Javascript-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/Javascript-program-level/templates.yaml
new file mode 100644
index 000000000..a1ba52fd4
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/Javascript-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: Python-program-level
+templates:
+  5f718b1c-7514-488f-99ed-4da5da70e103: !Template
+    answer_choices: null
+    id: 5f718b1c-7514-488f-99ed-4da5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in Javascript:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: soljs
+    reference: ''
+  5b218b2e-7514-488f-99ed-3cb6da70e103: !Template
+    answer_choices: null
+    id: 5b218b2e-7514-488f-99ed-3cb6da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in JS?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovejs
+    reference: ''
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml
new file mode 100644
index 000000000..1c09733d5
--- /dev/null
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml
@@ -0,0 +1,39 @@
+dataset: codeparrot/xlcost-text-to-code
+subset: Python-program-level
+templates:
+  5f718b9d-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5f718b9d-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      Solution in php:
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: solc
+    reference: ''
+  5b558b2e-7514-488f-99ed-3ca5da70e103: !Template
+    answer_choices: null
+    id: 5b558b2e-7514-488f-99ed-3ca5da70e103
+    jinja: '"{{ text }}"
+
+      How can the above be solved in PHP?
+
+      |||
+      {{ code_clean }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      languages:
+      - en
+      metrics:
+      - Other
+      original_task: true
+    name: abovec
+    reference: ''

From 4a0bb91803770bf360d4786ae6bd9c89ee51ec64 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 23:20:08 +0200
Subject: [PATCH 18/36] Add

---
 .../top_level/templates.yaml                  | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
new file mode 100644
index 000000000..e215f17c6
--- /dev/null
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -0,0 +1,44 @@
+dataset: teven/code_docstring_corpus
+subset: top_level
+templates:
+  5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
+    id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
+    answer_choices: null
+    jinja: 'Complete the below
+
+    {{decl}}
+        {{desc}}
+    ||| 
+    
+    {{bodies}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: complete
+    reference: ''
+  2c9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
+    id: 2c9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
+    answer_choices: null
+    jinja: "I wrote the below code
+    
+    {{bodies}}
+
+
+    What's a good function header?
+
+    |||
+    
+    {{decl}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: funcname
+    reference: ''

From 79373d15dd0f16c2d438f9a37dc8cedd66a991ed Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Wed, 24 Aug 2022 23:25:02 +0200
Subject: [PATCH 19/36] Add

---
 .../teven/code_docstring_corpus/top_level/templates.yaml      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index e215f17c6..e607da84a 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -7,7 +7,9 @@ templates:
     jinja: 'Complete the below
 
     {{decl}}
-        {{desc}}
+
+        """{{desc}}"""
+
     ||| 
     
     {{bodies}}'

From 575f025e593a519b453a8a71cc1e839081a750ad Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:06:38 +0200
Subject: [PATCH 20/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml   | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index e607da84a..438f03c7e 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -8,11 +8,10 @@ templates:
 
     {{decl}}
 
-        """{{desc}}"""
+    \t"""{{desc}}"""
 
-    ||| 
-    
-    {{bodies}}'
+    |||
+    \t{{bodies}}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 88cde9f795781b72eb48a162b2950e87f0524310 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:09:09 +0200
Subject: [PATCH 21/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 438f03c7e..d092734f6 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -8,10 +8,10 @@ templates:
 
     {{decl}}
 
-    \t"""{{desc}}"""
+      """{{desc}}"""
 
     |||
-    \t{{bodies}}'
+      {{bodies}}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 46070a0ade7a71f9a9e9c7aca4e64760351b6bf9 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:11:46 +0200
Subject: [PATCH 22/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index d092734f6..e611d76bb 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,14 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: 'Complete the below
-
-    {{decl}}
-
-      """{{desc}}"""
-
-    |||
-      {{bodies}}'
+    jinja: 'Complete the below\n{{decl}}\t"""{{desc}}"""\n|||{{bodies}}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From ab848d30fbbe4c83ed8bf02eef3ddffcbef58069 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:12:46 +0200
Subject: [PATCH 23/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index e611d76bb..dd078f52f 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: 'Complete the below\n{{decl}}\t"""{{desc}}"""\n|||{{bodies}}'
+    jinja: 'Complete the below\n{{decl}}\t{{desc}}\n|||{{bodies}}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From fa971b4756302c141c4b129c14552faab6a68fd4 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:13:37 +0200
Subject: [PATCH 24/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index dd078f52f..c2adeda55 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: 'Complete the below\n{{decl}}\t{{desc}}\n|||{{bodies}}'
+    jinja: 'Complete the below\\n{{decl}}\\t{{desc}}\\n|||{{bodies}}'
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 04f4a916e35249c00b4317e8015ee9efdcd7bdfd Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:14:17 +0200
Subject: [PATCH 25/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index c2adeda55..4526ceed8 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: 'Complete the below\\n{{decl}}\\t{{desc}}\\n|||{{bodies}}'
+    jinja: "Complete the below\n{{decl}}\t{{desc}}\n|||{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From cebbe615a0a6784296763821ca4ba1bf8c47f806 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:16:27 +0200
Subject: [PATCH 26/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 4526ceed8..4aa5141a6 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: "Complete the below\n{{decl}}\t{{desc}}\n|||{{bodies}}"
+    jinja: "Complete the below\n{{decl}}\t{{desc | replace('\n', '\n\t')}}\n|||{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:
@@ -19,6 +19,7 @@ templates:
     answer_choices: null
     jinja: "I wrote the below code
     
+
     {{bodies}}
 
 

From 12396129be319bfc759a07ba61273716afbf85d1 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:17:04 +0200
Subject: [PATCH 27/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 4aa5141a6..1cf861cc8 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: "Complete the below\n{{decl}}\t{{desc | replace('\n', '\n\t')}}\n|||{{bodies}}"
+    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n\t')}}'''\n|||{{bodies| replace('\n', '\n\t')}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 8f6468b2b7e95c99699da9cdf274cbacfeb8040e Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:17:44 +0200
Subject: [PATCH 28/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 1cf861cc8..abb5c9d79 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n\t')}}'''\n|||{{bodies| replace('\n', '\n\t')}}"
+    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n\t')}}'''\n|||{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 4d3d840f54aef8956af18af6df40a71d79abdbae Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:19:43 +0200
Subject: [PATCH 29/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index abb5c9d79..5a1897eff 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n\t')}}'''\n|||{{bodies}}"
+    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n    ')}}'''\n|||{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 1805c101ea2b1ee692ba58170793507bf20d8a31 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:21:14 +0200
Subject: [PATCH 30/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 5a1897eff..2c94375bf 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -4,7 +4,7 @@ templates:
   5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8: !Template
     id: 5d6956f9-a0e2-4fbb-aee8-b26ead2cf6b8
     answer_choices: null
-    jinja: "Complete the below\n{{decl}}\t'''{{desc | replace('\n', '\n    ')}}'''\n|||{{bodies}}"
+    jinja: "Complete the below\n{{decl}}\n    '''{{desc | replace('\n', '\n    ')}}'''\n|||{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From c80a1a39ce47f452576b1a83ee9669a06e9f4c7a Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:24:09 +0200
Subject: [PATCH 31/36] Fix'

---
 .../code_docstring_corpus/top_level/templates.yaml  | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 2c94375bf..ba30645d6 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -37,3 +37,16 @@ templates:
       original_task: true
     name: funcname
     reference: ''
+  4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
+    id: 4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
+    answer_choices: null
+    jinja: "{{decl}}|||\n    \"\"\"{{desc | replace('\n', '\n    ') | replace('\'', '')}}\"\"\"\n{{bodies}}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: funcname
+    reference: ''

From cfe4070ae72e26a5df0acbeb647e9b10683b905f Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:24:46 +0200
Subject: [PATCH 32/36] Fix'

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index ba30645d6..6db189892 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -40,7 +40,7 @@ templates:
   4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
     id: 4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
     answer_choices: null
-    jinja: "{{decl}}|||\n    \"\"\"{{desc | replace('\n', '\n    ') | replace('\'', '')}}\"\"\"\n{{bodies}}"
+    jinja: "{{decl}}|||\n    \"\"\"{{desc | replace('\n', '\n    ') | replace(''', '')}}\"\"\"\n{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From c2d3c7d2a408121c9131700f2e6fc6bcb180b00f Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:25:23 +0200
Subject: [PATCH 33/36] Fix

---
 .../teven/code_docstring_corpus/top_level/templates.yaml        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index 6db189892..d182bf19a 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -40,7 +40,7 @@ templates:
   4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1: !Template
     id: 4d9d55bc-eb77-4e7c-af6e-3d15b79d6cf1
     answer_choices: null
-    jinja: "{{decl}}|||\n    \"\"\"{{desc | replace('\n', '\n    ') | replace(''', '')}}\"\"\"\n{{bodies}}"
+    jinja: "{{decl}}|||\n    \"\"\"{{desc | replace('\n', '\n    ') | replace(\"'\", '')}}\"\"\"\n{{bodies}}"
     metadata: !TemplateMetadata
       choices_in_prompt: true
       languages:

From 5cde25b378df7ca9e108371cd861c1d105613280 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 10:37:47 +0200
Subject: [PATCH 34/36] fixes

---
 .../xlcost-text-to-code/Java-program-level/templates.yaml     | 2 +-
 .../xlcost-text-to-code/PHP-program-level/templates.yaml      | 4 ++--
 .../teven/code_docstring_corpus/top_level/templates.yaml      | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
index 10732cd36..6069b69fa 100644
--- a/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/Java-program-level/templates.yaml
@@ -17,7 +17,7 @@ templates:
       metrics:
       - Other
       original_task: true
-    name: solc
+    name: soljava
     reference: ''
   5b218b2e-7579-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
diff --git a/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml b/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml
index 1c09733d5..45a4c93cf 100644
--- a/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml
+++ b/promptsource/templates/codeparrot/xlcost-text-to-code/PHP-program-level/templates.yaml
@@ -17,7 +17,7 @@ templates:
       metrics:
       - Other
       original_task: true
-    name: solc
+    name: solphp
     reference: ''
   5b558b2e-7514-488f-99ed-3ca5da70e103: !Template
     answer_choices: null
@@ -35,5 +35,5 @@ templates:
       metrics:
       - Other
       original_task: true
-    name: abovec
+    name: abovephp
     reference: ''
diff --git a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
index d182bf19a..dca8e893a 100644
--- a/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
+++ b/promptsource/templates/teven/code_docstring_corpus/top_level/templates.yaml
@@ -48,5 +48,5 @@ templates:
       metrics:
       - Accuracy
       original_task: true
-    name: funcname
+    name: funccont
     reference: ''

From f52ea347cdffce1124c9bde944469e7ded8c5b9b Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Thu, 25 Aug 2022 17:43:52 +0200
Subject: [PATCH 35/36] Add Ps

---
 .../teven/code_contests/templates.yaml        | 72 +++++++++++++++++++
 1 file changed, 72 insertions(+)

diff --git a/promptsource/templates/teven/code_contests/templates.yaml b/promptsource/templates/teven/code_contests/templates.yaml
index 760759e22..c5d0fd377 100644
--- a/promptsource/templates/teven/code_contests/templates.yaml
+++ b/promptsource/templates/teven/code_contests/templates.yaml
@@ -56,3 +56,75 @@ templates:
       original_task: true
     name: abovesol
     reference: ''
+  9dcd29d1-cad0-4d2f-a7cc-120df233571c: !Template
+    id: 9dcd29d1-cad0-4d2f-a7cc-120df233571c
+    answer_choices: null
+    jinja: '{{description}}
+    
+    Solve the task in {{language}}.
+    
+    |||
+
+    {{solution}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: soltask
+    reference: ''
+  3dcf99d1-cad0-4d2f-a7cc-120df233571c: !Template
+    id: 3dcf99d1-cad0-4d2f-a7cc-120df233571c
+    answer_choices: null
+    jinja: '{{description}}
+    
+    Using {{language | lower}} can you solve the prior task?
+    
+    |||
+
+    {{solution}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: priortask
+    reference: ''
+  2dcf99d1-cad0-4d2f-a7cc-120df233571c: !Template
+    id: 2dcf99d1-cad0-4d2f-a7cc-120df233571c
+    answer_choices: null
+    jinja: '{{description}}
+    
+    {{solution[:5]}}|||{{solution[5:]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: contsol
+    reference: ''
+  9dcf99d5-cad0-4d2f-a7cc-120df233571c: !Template
+    id: 9dcf99d5-cad0-4d2f-a7cc-120df233571c
+    answer_choices: null
+    jinja: '{{language}} solution for "{{description}}":
+
+
+    
+    
+    |||
+    {{solution}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: solfor
+    reference: ''

From 5654fb0b19b54f6dc6e62d65fdc65703d260db17 Mon Sep 17 00:00:00 2001
From: Muennighoff <n.muennighoff@gmail.com>
Date: Sat, 27 Aug 2022 17:10:48 +0200
Subject: [PATCH 36/36] Add strip conn option

---
 promptsource/templates.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/promptsource/templates.py b/promptsource/templates.py
index fec4e5e4b..0255e40c6 100644
--- a/promptsource/templates.py
+++ b/promptsource/templates.py
@@ -378,12 +378,13 @@ def get_fixed_answer_choices_list(self):
         else:
             return None
 
-    def apply(self, example, truncate=True, highlight_variables=False) -> Tuple[str, List[str]]:
+    def apply(self, example, truncate=True, strip_connection=True, highlight_variables=False) -> Tuple[str, List[str]]:
         """
         Creates a prompt by applying this template to an example
 
         :param example: the dataset example to create a prompt for
         :param truncate: if True, example fields will be truncated to TEXT_VAR_LENGTH chars
+        :param strip_connection: if True, strips the connection between input & target
         :param highlight_variables: highlight the added variables
         :return: tuple of a string and a list of strings, for input and targets
         """
@@ -414,7 +415,10 @@ def apply(self, example, truncate=True, highlight_variables=False) -> Tuple[str,
 
         # Splits on the separator, and then replaces back any occurrences of the
         # separator in the original example
-        parts = [self._unescape_pipe(part).strip() for part in rendered_example.split("|||")]
+        if strip_connection:
+            parts = [self._unescape_pipe(part).strip() for part in rendered_example.split("|||")]
+        else:
+            parts = [self._unescape_pipe(part) for part in rendered_example.split("|||")]
         if parts == [""]:
             # Handles the case of blank results
             # Example: `tydiqa` where prompts are conditionned on the language and thus most of the time will return a blank result
@@ -422,7 +426,16 @@ def apply(self, example, truncate=True, highlight_variables=False) -> Tuple[str,
         if len(parts) < 2:
             raise ValueError("Prompt did not produce an input and at least one target.")
 
-        return parts[0], parts[1:]
+        if strip_connection:
+            return parts[0], parts[1:]
+        else:
+            # Remove double whitespace
+            if parts[0][-1] == " " and all(p[0] == " " for p in parts[1:]):
+                parts[0] = parts[0][:-1]
+            # Leave the connection between input & target unstripped
+            return parts[0].lstrip(), [p.rstrip() for p in parts[1:]]
+
+        
 
     pipe_protector = "3ed2dface8203c4c9dfb1a5dc58e41e0"