diff --git a/promptsource/templates.py b/promptsource/templates.py index 2b9eb341e..39c8921a0 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -27,7 +27,18 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"} +INCLUDED_USERS = { + "Zaid", + "craffel", + "GEM", + "aps", + "khalidalt", + "shanya", + "rbawden", + "BigScienceBiasEval", + "gsarti", + "tau", + } # These are the metrics with which templates can be tagged METRICS = { diff --git a/promptsource/templates/tau/scrolls/contract_nli/templates.yaml b/promptsource/templates/tau/scrolls/contract_nli/templates.yaml new file mode 100644 index 000000000..40c789ab0 --- /dev/null +++ b/promptsource/templates/tau/scrolls/contract_nli/templates.yaml @@ -0,0 +1,63 @@ +dataset: tau/scrolls +subset: contract_nli +templates: + 243c496c-4cae-46d7-b94d-91c946fbd98c: !Template + answer_choices: Not mentioned ||| Entailment ||| Contradiction + id: 243c496c-4cae-46d7-b94d-91c946fbd98c + jinja: 'The following is an NDA: + + {{input.split("\n\n", 1)[1]}} + + Here is a statement about that NDA: {{input.split("\n\n", 1)[0]}} + + Based on this NDA, is the statement "Not mentioned", an "Entailment", or a "Contradiction"? + ||| {{output}}' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Accuracy + - Other + original_task: true + name: statement after + reference: '' + 8082ee08-d6ae-4d5d-9fbf-3dae04e9b7ec: !Template + answer_choices: null + id: 8082ee08-d6ae-4d5d-9fbf-3dae04e9b7ec + jinja: '{{input.split("\n\n", 1)[0]}} + + Please write a non-disclosure agreement with the relation of "{{output}}" to + the previous statement: ||| {{input.split("\n\n", 1)[1]}} + + ' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: write_nda + reference: '' + b1ba1f3b-7eba-4bf2-aa21-a2d899d05e40: !Template + answer_choices: Not mentioned ||| Entailment ||| Contradiction + id: b1ba1f3b-7eba-4bf2-aa21-a2d899d05e40 + jinja: 'The following is a statement followed by a non-disclosure agreement: + + {{input}} + + Based on this NDA, is the statement "Not mentioned", an "Entailment", or a "Contradiction"? + ||| {{output}} + + ' + metadata: !TemplateMetadata + choices_in_prompt: true + languages: + - en + metrics: + - Other + original_task: true + name: nli_labelsgiven + reference: '' diff --git a/promptsource/templates/tau/scrolls/gov_report/templates.yaml b/promptsource/templates/tau/scrolls/gov_report/templates.yaml new file mode 100644 index 000000000..f91e58c3f --- /dev/null +++ b/promptsource/templates/tau/scrolls/gov_report/templates.yaml @@ -0,0 +1,87 @@ +dataset: tau/scrolls +subset: gov_report +templates: + 2edf0297-e3d4-4870-bd95-178411e52e09: !Template + answer_choices: null + id: 2edf0297-e3d4-4870-bd95-178411e52e09 + jinja: 'Summary: {{ output }} + + Full Document: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - ROUGE + - BLEU + original_task: false + name: summary_doc + reference: '' + 3bd082cb-4e28-4eb7-1ea2-dd03f1f86219: !Template + answer_choices: null + id: 3bd082cb-4e28-4eb7-1ea2-dd03f1f86219 + jinja: 'Read the following document: + + {{ output }} + + The following is a concise summary of the document: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: read_following + reference: '' + 52ffecee-4298-4611-a3dd-cc0ee3856259: !Template + answer_choices: null + id: 52ffecee-4298-4611-a3dd-cc0ee3856259 + jinja: 'The following is a summary taken from a government report. + + Summary: {{ output }} + + The contents of that report were likely as follows: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: reconstruct_report + reference: '' + af4d550e-54b8-471e-97af-2b2c50a1382e: !Template + answer_choices: null + id: af4d550e-54b8-471e-97af-2b2c50a1382e + jinja: 'Document: {{ input }} + + Summary: ||| {{ output }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: true + name: doc_summary + reference: '' + af4d550e-54b8-471e-97af-2b2c50a1382e: !Template + answer_choices: null + id: af4d550e-54b8-471e-97af-2b2c50a1382e + jinja: 'Here is a government report: {{ input }} + + I will now summarize it: ||| {{ output }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: true + name: summarize_report + reference: '' diff --git a/promptsource/templates/tau/scrolls/qmsum/templates.yaml b/promptsource/templates/tau/scrolls/qmsum/templates.yaml new file mode 100644 index 000000000..aa5dde228 --- /dev/null +++ b/promptsource/templates/tau/scrolls/qmsum/templates.yaml @@ -0,0 +1,74 @@ +dataset: tau/scrolls +subset: qmsum +templates: + 2edf0197-e3d4-4870-bd95-178411e52e09: !Template + answer_choices: null + id: 2edf0197-e3d4-4870-bd95-178411e52e09 + jinja: 'This is a summary of a meeting based on a query. + + Summary: {{ output }} + + The query given and the original meeting transcript\ are as follows: ||| {{ + input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - ROUGE + - BLEU + original_task: false + name: write_meeting + reference: '' + 8c3d956e-320b-4800-8dd1-552330b7f3dc: !Template + answer_choices: null + id: 8c3d956e-320b-4800-8dd1-552330b7f3dc + jinja: 'Given the following meeting transcript: {{ input.split("?", 1)[1][1:] + }} + + What is a question you could ask about the meeting? ||| {{ input.split("?", + 1)[0] }}?' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: Generate question + reference: '' + af4d550e-54b8-471e-97af-2b2c50a1382e: !Template + answer_choices: null + id: af4d550e-54b8-471e-97af-2b2c50a1382e + jinja: 'The following is a question followed by a meeting transcript: + + {{ input }} + + How would you best summarize the meeting based on the question? ||| {{ output + }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: true + name: summarize_by_query + reference: '' + cd5f9c35-1074-41bd-b362-d152b8d51894: !Template + answer_choices: null + id: cd5f9c35-1074-41bd-b362-d152b8d51894 + jinja: '{{ input.split("?", 1)[1][1:] }} + + {{ input.split("?", 1)[0] }}? + + Answer: ||| {{ output }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: [] + metrics: [] + original_task: false + name: question_last + reference: '' diff --git a/promptsource/templates/tau/scrolls/summ_screen_fd/templates.yaml b/promptsource/templates/tau/scrolls/summ_screen_fd/templates.yaml new file mode 100644 index 000000000..9a50e093b --- /dev/null +++ b/promptsource/templates/tau/scrolls/summ_screen_fd/templates.yaml @@ -0,0 +1,75 @@ +dataset: tau/scrolls +subset: summ_screen_fd +templates: + 2edf0197-e3d4-4870-bd95-178411e52e09: !Template + answer_choices: null + id: 2edf0197-e3d4-4870-bd95-178411e52e09 + jinja: 'Given the following television episode recap, please write a transcript of the episode. + + Recap: {{ output }} + + Transcript: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - ROUGE + - BLEU + original_task: false + name: write_transcript + reference: '' + 3bd082cb-4e28-4eb7-9fa2-dd03f1f86219: !Template + answer_choices: null + id: 3bd082cb-4e28-4eb7-9fa2-dd03f1f86219 + jinja: 'Here is a summary of a TV episode: + + {{ output }} + + The following is the full screenplay of that TV episode: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: provide screenplay + reference: '' + 52ffecee-4298-4611-a3dd-cc0ee3856259: !Template + answer_choices: null + id: 52ffecee-4298-4611-a3dd-cc0ee3856259 + jinja: 'I will provide a TV episode synopsis below. + + Synopsis: {{ output }} + + I will now start writing that TV episode from scratch: ||| {{ input }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: false + name: write_from_synopsis + reference: '' + af4d550e-54b8-471e-97af-2b2c50a1382e: !Template + answer_choices: null + id: af4d550e-54b8-471e-97af-2b2c50a1382e + jinja: 'The following is a television episode transcript: + + {{ input }} + + How would you best summarize the above episode? ||| {{ output }}' + metadata: !TemplateMetadata + choices_in_prompt: false + languages: + - en + metrics: + - BLEU + - ROUGE + original_task: true + name: summarize episode + reference: ''