{"taskset": {"n_runnable": 0, "name": "SimpleQA", "url_data": "https://www.kaggle.com/benchmarks/openai/simpleqa", "catalog_id": "simpleqa", "owner_type": "lab", "cited_by": [], "taskset": "simpleqa", "domain": "reasoning-knowledge", "task_kind": "A benchmark from OpenAI designed to evaluate short-form factuality in large language models.", "environment": "none", "grading": "mixed", "path": "datasets/simpleqa", "n_single_container": 4326, "n_tasks": 4326, "domain_raw": "knowledge", "sample_instruction": "Answer the following factual question accurately and concisely.\n\n**Question:** Who received the IEEE Frank Rosenblatt Award in 2010?\n\nWrite your answer to `/workspace/answer.txt`. Your answer should be:\n- A direct, factual response to the question\n- As concise as possible while being complete\n- Without unnecessary hedging or qualifications\n\n**Important:**\n- You should ONLY interact with the environment provided to you AND NEVER ASK FOR HUMAN HELP.\n- Focus on accuracy - if you're uncertain, it's better to acknowledge that than to guess incorrectly.\n- The answer file should contain your response as plain text.\n", "owner_org": "OpenAI", "url_repo": null, "url_paper": "https://arxiv.org/abs/2411.04368", "license": null, "languages": [], "difficulties": {"medium": 4326}, "categories": ["factuality"]}, "tasks": [{"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-0", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "number"], "task": "simpleqa-10", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-100", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1000", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1001", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "general", "unknown"], "task": "simpleqa-1002", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1003", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1004", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1005", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1006", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1007", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "person"], "task": "simpleqa-1008", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1009", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "other"], "task": "simpleqa-101", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1010", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1011", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "other"], "task": "simpleqa-1012", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "number"], "task": "simpleqa-1013", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1014", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1015", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1016", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1017", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1018", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "person"], "task": "simpleqa-1019", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-102", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "number"], "task": "simpleqa-1020", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1021", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1022", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1023", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1024", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "general", "unknown"], "task": "simpleqa-1025", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "person"], "task": "simpleqa-1026", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1027", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "place"], "task": "simpleqa-1028", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1029", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "number"], "task": "simpleqa-103", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "number"], "task": "simpleqa-1030", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "person"], "task": "simpleqa-1031", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "place"], "task": "simpleqa-1032", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "other"], "task": "simpleqa-1033", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1034", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1035", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1036", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-1037", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "number"], "task": "simpleqa-1038", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1039", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "number"], "task": "simpleqa-104", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "other"], "task": "simpleqa-1040", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1041", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1042", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1043", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-1044", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "person"], "task": "simpleqa-1045", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1046", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "place"], "task": "simpleqa-1047", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1048", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "date"], "task": "simpleqa-1049", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "person"], "task": "simpleqa-105", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1050", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1051", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "person"], "task": "simpleqa-1052", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1053", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1054", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1055", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1056", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "number"], "task": "simpleqa-1057", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "number"], "task": "simpleqa-1058", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "other"], "task": "simpleqa-1059", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-106", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "number"], "task": "simpleqa-1060", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1061", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "date"], "task": "simpleqa-1062", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "other"], "task": "simpleqa-1063", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "date"], "task": "simpleqa-1064", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "other"], "task": "simpleqa-1065", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1066", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "number"], "task": "simpleqa-1067", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1068", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "number"], "task": "simpleqa-1069", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-107", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "other"], "task": "simpleqa-1070", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1071", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1072", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "number"], "task": "simpleqa-1073", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1074", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "other"], "task": "simpleqa-1075", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "number"], "task": "simpleqa-1076", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1077", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1078", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1079", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "other"], "task": "simpleqa-108", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1080", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "other"], "task": "simpleqa-1081", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "other"], "task": "simpleqa-1082", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-1083", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "number"], "task": "simpleqa-1084", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1085", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "other"], "task": "simpleqa-1086", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "other"], "task": "simpleqa-1087", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1088", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1089", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-109", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "general", "unknown"], "task": "simpleqa-1090", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1091", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "person"], "task": "simpleqa-1092", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "general", "unknown"], "task": "simpleqa-1093", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1094", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1095", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1096", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1097", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "other"], "task": "simpleqa-1098", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "other"], "task": "simpleqa-1099", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "date"], "task": "simpleqa-11", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "place"], "task": "simpleqa-110", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1100", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1101", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "number"], "task": "simpleqa-1102", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1103", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "number"], "task": "simpleqa-1104", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-1105", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "place"], "task": "simpleqa-1106", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "place"], "task": "simpleqa-1107", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1108", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1109", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-111", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1110", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "person"], "task": "simpleqa-1111", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1112", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1113", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1114", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1115", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1116", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1117", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "other"], "task": "simpleqa-1118", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "person"], "task": "simpleqa-1119", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "general", "unknown"], "task": "simpleqa-112", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "date"], "task": "simpleqa-1120", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "number"], "task": "simpleqa-1121", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "history", "date"], "task": "simpleqa-1122", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1123", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1124", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1125", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "person"], "task": "simpleqa-1126", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1127", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "other"], "task": "simpleqa-1128", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "number"], "task": "simpleqa-1129", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "date"], "task": "simpleqa-113", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1130", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-1131", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "place"], "task": "simpleqa-1132", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "number"], "task": "simpleqa-1133", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1134", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "person"], "task": "simpleqa-1135", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "music", "other"], "task": "simpleqa-1136", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "place"], "task": "simpleqa-1137", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1138", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1139", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "place"], "task": "simpleqa-114", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1140", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1141", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "other"], "task": "simpleqa-1142", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1143", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1144", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "place"], "task": "simpleqa-1145", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "other"], "task": "simpleqa-1146", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1147", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1148", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1149", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "number"], "task": "simpleqa-115", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "number"], "task": "simpleqa-1150", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1151", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "date"], "task": "simpleqa-1152", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1153", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "other"], "task": "simpleqa-1154", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "place"], "task": "simpleqa-1155", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "person"], "task": "simpleqa-1156", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "number"], "task": "simpleqa-1157", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "place"], "task": "simpleqa-1158", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "date"], "task": "simpleqa-1159", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "number"], "task": "simpleqa-116", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "number"], "task": "simpleqa-1160", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1161", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1162", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1163", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "sports", "person"], "task": "simpleqa-1164", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "date"], "task": "simpleqa-1165", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "video-games", "other"], "task": "simpleqa-1166", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1167", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "tv-shows", "person"], "task": "simpleqa-1168", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "geography", "date"], "task": "simpleqa-1169", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-117", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "person"], "task": "simpleqa-1170", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "place"], "task": "simpleqa-1171", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "other"], "task": "simpleqa-1172", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "other", "date"], "task": "simpleqa-1173", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "art", "number"], "task": "simpleqa-1174", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "person"], "task": "simpleqa-1175", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "politics", "date"], "task": "simpleqa-1176", "taskset": "simpleqa"}, {"agent_timeout": 3000, "category": "factuality", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["simpleqa", "factuality", "science-and-technology", "date"], "task": "simpleqa-1177", "taskset": "simpleqa"}], "next": "simpleqa-1177"}