From 3ecf9b47e5698c7e67f3b10807071cbca95f79fb Mon Sep 17 00:00:00 2001 From: Xu Song Date: Mon, 6 May 2024 15:31:42 +0800 Subject: [PATCH] Fix AGIEval chinese sets (#972) * [Fix] Fix AGIEval chinese sets * Create agieval_gen_617738.py * [Fix] Fix AGIEval chinese sets * Restore agieval_gen_64afd3.py * Update agieval_gen.py * Create agieval_mixed_0fa998.py * Update agieval_mixed.py --- configs/datasets/agieval/agieval_gen.py | 2 +- .../datasets/agieval/agieval_gen_617738.py | 209 +++++++++++++++++ configs/datasets/agieval/agieval_mixed.py | 2 +- .../datasets/agieval/agieval_mixed_0fa998.py | 213 ++++++++++++++++++ 4 files changed, 424 insertions(+), 2 deletions(-) create mode 100644 configs/datasets/agieval/agieval_gen_617738.py create mode 100644 configs/datasets/agieval/agieval_mixed_0fa998.py diff --git a/configs/datasets/agieval/agieval_gen.py b/configs/datasets/agieval/agieval_gen.py index 6af2dc6ae..64808e62f 100644 --- a/configs/datasets/agieval/agieval_gen.py +++ b/configs/datasets/agieval/agieval_gen.py @@ -1,4 +1,4 @@ from mmengine.config import read_base with read_base(): - from .agieval_gen_64afd3 import agieval_datasets # noqa: F401, F403 + from .agieval_gen_617738 import agieval_datasets # noqa: F401, F403 diff --git a/configs/datasets/agieval/agieval_gen_617738.py b/configs/datasets/agieval/agieval_gen_617738.py new file mode 100644 index 000000000..f68d5d7cf --- /dev/null +++ b/configs/datasets/agieval/agieval_gen_617738.py @@ -0,0 +1,209 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator +from opencompass.utils.text_postprocessors import first_option_postprocess, first_capital_postprocess_multi + +agieval_reader_cfg = dict( + input_columns=['question', 'options'], output_column='label') + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + 'gaokao-physics', + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] +agieval_chinese_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'gaokao-mathcloze', + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_english_sets = [ + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', + 'math', +] +agieval_gaokao_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', +] + +agieval_datasets = [] +for _name in agieval_single_choice_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict( + type=first_option_postprocess, options='ABCDE')) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_multiple_choices_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict( + role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AccEvaluator), + pred_postprocessor=dict(type=first_capital_postprocess_multi)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_cloze_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role='HUMAN', prompt=f'{{question}}\n{_hint}')])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=agieval_reader_cfg, + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _item in agieval_datasets: + _name = _item['name'] + _intro = { + 'gaokao-chinese': + '以下是一道中国高考语文选择题，请选择正确的答案。', + 'gaokao-english': + '以下是一道中国高考英语选择题，请选择正确的答案。', + 'gaokao-geography': + '以下是一道中国高考地理选择题，请选择正确的答案。', + 'gaokao-history': + '以下是一道中国高考历史选择题，请选择正确的答案。', + 'gaokao-biology': + '以下是一道中国高考生物选择题，请选择正确的答案。', + 'gaokao-chemistry': + '以下是一道中国高考化学选择题，请选择正确的答案。', + 'gaokao-physics': + '以下是一道中国高考物理选择题，请选择正确的答案。', + 'gaokao-mathqa': + '以下是一道中国高考数学选择题，请选择正确的答案。', + 'logiqa-zh': + '以下是一道中国公务员考试题，请选择正确的答案。', + 'lsat-ar': + 'The following is a LSAT Analytical Reasoning question. Please select the correct answer.', + 'lsat-lr': + 'The following is a LSAT Logical Reasoning question. Please select the correct answer.', + 'lsat-rc': + 'The following is a LSAT Reading Comprehension question. Please select the correct answer.', + 'logiqa-en': + 'The following is a Logic Reasoning question. Please select the correct answer.', + 'sat-math': + 'The following is a SAT Math question. Please select the correct answer.', + 'sat-en': + 'The following is a SAT English question. Please select the correct answer.', + 'sat-en-without-passage': + 'The following is a SAT English question. Please select the correct answer.', + 'aqua-rat': + 'The following is a AQUA-RAT question. Please select the correct answer.', + 'jec-qa-kd': + '以下是一道中国司法考试基础知识题，请选择正确的答案。', + 'jec-qa-ca': + '以下是一道中国司法考试案例分析题，请选择正确的答案。', + 'gaokao-mathcloze': + '以下是一道中国高考数学填空题，请填入正确的答案。', + 'math': + 'The following is a Math question. Please select the correct answer.', + }[_name] + _templates = _item['infer_cfg']['prompt_template']['template'] + _templates['round'][0][ + 'prompt'] = _intro + '\n' + _templates['round'][0]['prompt'] + +del _item, _intro, _templates, _name, _hint, agieval_infer_cfg, agieval_eval_cfg diff --git a/configs/datasets/agieval/agieval_mixed.py b/configs/datasets/agieval/agieval_mixed.py index a3e9ef58e..18259efc5 100644 --- a/configs/datasets/agieval/agieval_mixed.py +++ b/configs/datasets/agieval/agieval_mixed.py @@ -1,4 +1,4 @@ from mmengine.config import read_base with read_base(): - from .agieval_mixed_2f14ad import agieval_datasets # noqa: F401, F403 + from .agieval_mixed_0fa998 import agieval_datasets # noqa: F401, F403 diff --git a/configs/datasets/agieval/agieval_mixed_0fa998.py b/configs/datasets/agieval/agieval_mixed_0fa998.py new file mode 100644 index 000000000..5c5b40b86 --- /dev/null +++ b/configs/datasets/agieval/agieval_mixed_0fa998.py @@ -0,0 +1,213 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import PPLInferencer, GenInferencer +from opencompass.openicl.icl_evaluator import AccEvaluator +from opencompass.datasets import AGIEvalDataset_v2, AGIEvalEvaluator, AGIEvalEvaluator_mcq +from opencompass.utils.text_postprocessors import first_capital_postprocess_multi + +agieval_single_choice_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-mathqa', + 'logiqa-zh', + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', +] +agieval_multiple_choices_sets = [ + 'gaokao-physics', + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_cloze_sets = ['gaokao-mathcloze', 'math'] +agieval_chinese_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', + 'logiqa-zh', + 'gaokao-mathcloze', + 'jec-qa-kd', + 'jec-qa-ca', +] +agieval_english_sets = [ + 'lsat-ar', + 'lsat-lr', + 'lsat-rc', + 'logiqa-en', + 'sat-math', + 'sat-en', + 'sat-en-without-passage', + 'aqua-rat', + 'math', +] +agieval_gaokao_sets = [ + 'gaokao-chinese', + 'gaokao-english', + 'gaokao-geography', + 'gaokao-history', + 'gaokao-biology', + 'gaokao-chemistry', + 'gaokao-physics', + 'gaokao-mathqa', +] + +agieval_datasets = [] +for _name in agieval_single_choice_sets: + if _name in ['lsat-ar', 'lsat-lr', 'lsat-rc', 'aqua-rat']: + _options = ['A', 'B', 'C', 'D', 'E'] + else: + _options = ['A', 'B', 'C', 'D'] + if _name in agieval_chinese_sets: + _hint = '答案是：' + else: + _hint = 'The answer is ' + ]) + for label in _options + }), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=PPLInferencer, labels=_options)) + + agieval_eval_cfg = dict(evaluator=dict(type=AccEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'] + _options, + output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_multiple_choices_sets: + if _name in agieval_chinese_sets: + _hint = '答案是： ' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict(round=[ + dict(role='HUMAN', prompt=f'{{question}}\n{{options}}\n{_hint}') + ])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict( + evaluator=dict(type=AGIEvalEvaluator_mcq), + pred_postprocessor=dict(type=first_capital_postprocess_multi)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'], output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _name in agieval_cloze_sets: + if _name in agieval_chinese_sets: + _hint = '答案是：' + else: + _hint = 'The answer is ' + agieval_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + round=[dict(role='HUMAN', prompt=f'{{question}}{_hint}')])), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=1024)) + + agieval_eval_cfg = dict(evaluator=dict(type=AGIEvalEvaluator)) + + agieval_datasets.append( + dict( + type=AGIEvalDataset_v2, + path='./data/AGIEval/data/v1/', + name=_name, + abbr='agieval-' + _name, + setting_name='zero-shot', + reader_cfg=dict( + input_columns=['question', 'options'], output_column='label'), + infer_cfg=agieval_infer_cfg.copy(), + eval_cfg=agieval_eval_cfg.copy())) + +for _item in agieval_datasets: + _name = _item['name'] + _intro = { + 'gaokao-chinese': + '以下是一道中国高考语文选择题，请选择正确的答案。', + 'gaokao-english': + '以下是一道中国高考英语选择题，请选择正确的答案。', + 'gaokao-geography': + '以下是一道中国高考地理选择题，请选择正确的答案。', + 'gaokao-history': + '以下是一道中国高考历史选择题，请选择正确的答案。', + 'gaokao-biology': + '以下是一道中国高考生物选择题，请选择正确的答案。', + 'gaokao-chemistry': + '以下是一道中国高考化学选择题，请选择正确的答案。', + 'gaokao-physics': + '以下是一道中国高考物理选择题，请选择正确的答案。', + 'gaokao-mathqa': + '以下是一道中国高考数学选择题，请选择正确的答案。', + 'logiqa-zh': + '以下是一道中国公务员考试题，请选择正确的答案。', + 'lsat-ar': + 'The following is a LSAT Analytical Reasoning question. Please select the correct answer.', + 'lsat-lr': + 'The following is a LSAT Logical Reasoning question. Please select the correct answer.', + 'lsat-rc': + 'The following is a LSAT Reading Comprehension question. Please select the correct answer.', + 'logiqa-en': + 'The following is a Logic Reasoning question. Please select the correct answer.', + 'sat-math': + 'The following is a SAT Math question. Please select the correct answer.', + 'sat-en': + 'The following is a SAT English question. Please select the correct answer.', + 'sat-en-without-passage': + 'The following is a SAT English question. Please select the correct answer.', + 'aqua-rat': + 'The following is a AQUA-RAT question. Please select the correct answer.', + 'jec-qa-kd': + '以下是一道中国司法考试基础知识题，请选择正确的答案。', + 'jec-qa-ca': + '以下是一道中国司法考试案例分析题，请选择正确的答案。', + 'gaokao-mathcloze': + '以下是一道中国高考数学填空题，请填入正确的答案。', + 'math': + 'The following is a Math question. Please select the correct answer.', + }[_name] + _templates = _item['infer_cfg']['prompt_template']['template'] + + if _item['infer_cfg']['inferencer']['type'] == PPLInferencer: + for _label in _templates: + _templates[_label]['round'][0][ + 'prompt'] = _intro + '\n' + _templates[_label]['round'][0][ + 'prompt'] + else: + _templates['round'][0][ + 'prompt'] = _intro + '\n' + _templates['round'][0]['prompt'] + +del _item, _intro, _templates, _label, _name, _options, _hint, agieval_infer_cfg, agieval_eval_cfg