{"taskset": {"n_runnable": 0, "name": "ACE", "url_data": "https://huggingface.co/datasets/mercor/ACE", "catalog_id": "ace-mercor", "owner_type": "company", "cited_by": [], "taskset": "acebench-normal", "domain": "enterprise-ops", "task_kind": "evaluation criteria for DIY/food/shopping/gaming tasks", "environment": "live-web", "grading": "rubric", "path": "datasets/acebench-normal", "n_single_container": 823, "n_tasks": 823, "domain_raw": "consumer", "sample_instruction": "# Tool Usage Task\n\nYou are given a user question and a set of available tools. Call the correct tool(s) to answer the question.\n\n## Question\nuser: I'm looking for a list of high protein meals for dinner that include some vegetarian options.\nsystem: Could you please specify your preferred cuisine type, such as Italian or Asian?\nuser: I would like Asian cuisine, please.\n\n\n## Available Tools\n\n```json\n[\n  {\n    \"name\": \"ai.assistantModelSelector\",\n    \"description\": \"Selects the most effective machine learning model for predicting user behavior based on historical data and specified criteria.\",\n    \"arguments\": {\n      \"type\": \"object\",\n      \"properties\": {\n        \"data\": {\n          \"type\": \"object\",\n          \"properties\": {\n            \"userInteractions\": {\n              \"type\": \"array\",\n              \"items\": {\n                \"type\": \"object\",\n                \"properties\": {\n                  \"interactionType\": {\n                    \"type\": \"string\",\n                    \"description\": \"Type of interaction (e.g., click, view, purchase).\"\n                  },\n                  \"timestamp\": {\n                    \"type\": \"string\",\n                    \"format\": \"date-time\",\n                    \"description\": \"Timestamp of the interaction.\"\n                  }\n                },\n                \"required\": [\n                  \"interactionType\",\n                  \"timestamp\"\n                ]\n              },\n              \"description\": \"List of user interactions.\"\n            }", "owner_org": "Mercor", "url_repo": null, "url_paper": null, "license": "cc-by-4.0", "languages": [], "difficulties": {"medium": 823}, "categories": ["tool-use"]}, "tasks": [{"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_0", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_1", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_10", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_11", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_12", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_13", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_14", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_15", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_16", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_17", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_18", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_19", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_2", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_20", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_21", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_22", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_23", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_24", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_25", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_26", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_27", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_28", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_29", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_3", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_30", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_31", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_32", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_33", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_34", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_35", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_36", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_37", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_38", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_39", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_4", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_40", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_41", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_42", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_43", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_44", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_45", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_46", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_47", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_48", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_49", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_5", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_6", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_7", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_8", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_bool_9", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_0", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_1", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_10", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_11", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_12", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_13", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_14", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_15", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_16", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_17", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_18", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_19", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_2", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_20", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_21", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_22", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_23", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_24", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_25", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_26", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_27", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_28", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_29", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_3", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_30", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_31", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_32", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_33", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_34", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_35", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_36", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_37", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_38", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_39", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_4", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_40", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_41", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_42", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_43", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_44", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_45", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_46", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_47", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_48", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_49", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_5", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_6", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_7", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_8", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_enum_9", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_0", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_1", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_10", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_11", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_12", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_13", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_14", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_15", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_16", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_17", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_18", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_19", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_2", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_20", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_21", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_22", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_23", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_24", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_25", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_26", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_27", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_28", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_29", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_3", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_30", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_31", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_32", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_33", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_34", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_35", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_36", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_37", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_38", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_39", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_4", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_40", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_41", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_42", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_43", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_44", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_45", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_46", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_47", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_48", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_49", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_5", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_6", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_7", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_8", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_list_9", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_0", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_1", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_10", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_11", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_12", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_13", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_14", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_15", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_16", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_17", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_18", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_19", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_2", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_20", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_21", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_22", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_23", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_24", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_25", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_26", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_27", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_28", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_29", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_3", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_30", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_31", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_32", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_33", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_34", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_35", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_36", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_37", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_38", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_39", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_4", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_40", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_41", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_42", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_43", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_44", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_45", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_46", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_47", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_48", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_49", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_5", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_6", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_7", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_8", "taskset": "acebench-normal"}, {"agent_timeout": 300, "category": "tool-use", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": [], "task": "ace-bench_normal_atom_number_9", "taskset": "acebench-normal"}], "next": "ace-bench_normal_atom_number_9"}