{"taskset": {"n_runnable": 0, "name": null, "url_data": null, "catalog_id": null, "owner_type": null, "cited_by": null, "taskset": "lawbench", "domain": "legal", "task_kind": null, "environment": null, "grading": null, "path": "datasets/lawbench", "n_single_container": 1000, "n_tasks": 1000, "domain_raw": null, "sample_instruction": "You have 20 cases to process. You must answer ALL of them.\nCreate the file `/app/answer.jsonl` and write your output there.\n\nOUTPUT FORMAT (STRICT, JSON Lines / ndjson):\n\n1. `/app/answer.jsonl` must contain EXACTLY 20 non-empty lines.\n2. Each line must be a valid JSON object with EXACTLY two keys:\n   - \"id\": string\n   - \"answer\": string\n     No extra keys are allowed.\n3. The \"id\" must match the input id exactly.\n4. Output must be JSON lines only:\n   - No markdown fences (no ```).\n   - No headings, numbering, commentary, or extra text.\n   - No blank lines.\n5. Output language: Chinese.\n6. For each case, output ONLY the final answer text required by the question. Do not add explanations.\n7. Answers must be single-line strings. Do not include line breaks.\n   If the answer would normally include line breaks, replace them with a single space.\n8. Do not skip any case. If you cannot answer a case, output:\n   {\"id\":\"<id>\",\"answer\":\"\"}\n\nExample Output (format only; do not copy the placeholder text):\n{\"id\":\"0\",\"answer\":\"<\u8fd9\u91cc\u586b\u5199\u5bf9\u5e94\u9898\u76ee\u7684\u4e2d\u6587\u7b54\u6848>\"}\n{\"id\":\"1\",\"answer\":\"<\u8fd9\u91cc\u586b\u5199\u5bf9\u5e94\u9898\u76ee\u7684\u4e2d\u6587\u7b54\u6848>\"}\n\nINPUT CASES:\n{\"id\": \"0\", \"q\": \"\u56de\u7b54\u4ee5\u4e0b\u95ee\u9898\uff0c\u53ea\u9700\u76f4\u63a5\u7ed9\u51fa\u6cd5\u6761\u5185\u5bb9:\u4e0b\u9762\u662f\u4e00\u4e2a\u4f8b\u5b50:\\n\u6c11\u6cd5\u5546\u6cd5\u516c\u53f8\u6cd5\u7b2c\u516d\u5341\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\\n\u7b54\u6848:\u4e00\u4eba\u6709\u9650\u8d23\u4efb\u516c\u53f8\u7ae0\u7a0b\u7531\u80a1\u4e1c\u5236\u5b9a\u3002\\n\u8bf7\u4f60\u56de\u7b54\uff1a\\n\u6c11\u6cd5\u5546\u6cd5\u519c\u6c11\u4e13\u4e1a\u5408\u4f5c\u793e\u6cd5\u7b2c\u4e09\u5341\u4e09\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\"}\n{\"id\": \"1\", \"q\": \"\u56de\u7b54\u4ee5\u4e0b\u95ee\u9898\uff0c\u53ea\u9700\u76f4\u63a5\u7ed9\u51fa\u6cd5\u6761\u5185\u5bb9:\u4e0b\u9762\u662f\u4e00\u4e2a\u4f8b\u5b50:\\n\u6c11\u6cd5\u5546\u6cd5\u516c\u53f8\u6cd5\u7b2c\u516d\u5341\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\\n\u7b54\u6848:\u4e00\u4eba\u6709\u9650\u8d23\u4efb\u516c\u53f8\u7ae0\u7a0b\u7531\u80a1\u4e1c\u5236\u5b9a\u3002\\n\u8bf7\u4f60\u56de\u7b54\uff1a\\n\u6c11\u6cd5\u5546\u6cd5\u519c\u6c11\u4e13\u4e1a\u5408\u4f5c\u793e\u6cd5\u7b2c\u4e00\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\"}\n{\"id\": \"2\", \"q\": \"\u56de\u7b54\u4ee5\u4e0b\u95ee\u9898\uff0c\u53ea\u9700\u76f4\u63a5\u7ed9\u51fa\u6cd5\u6761\u5185\u5bb9:\u4e0b\u9762\u662f\u4e00\u4e2a\u4f8b\u5b50:\\n\u6c11\u6cd5\u5546\u6cd5\u516c\u53f8\u6cd5\u7b2c\u516d\u5341\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\\n\u7b54\u6848:\u4e00\u4eba\u6709\u9650\u8d23\u4efb\u516c\u53f8\u7ae0\u7a0b\u7531\u80a1\u4e1c\u5236\u5b9a\u3002\\n\u8bf7\u4f60\u56de\u7b54\uff1a\\n\u6c11\u6cd5\u5546\u6cd5\u519c\u6c11\u4e13\u4e1a\u5408\u4f5c\u793e\u6cd5\u7b2c\u516d\u5341\u4e94\u6761\u7684\u5185\u5bb9\u662f\u4ec0\u4e48\uff1f\"}\n{\"id\": \"3\", \"q\": \"\u56de\u7b54\u4ee5\u4e0b\u95ee\u9898\uff0c\u53ea\u9700\u76f4\u63a5\u7ed9\u51fa\u6cd5\u6761\u5185\u5bb9:\u4e0b\u9762\u662f\u4e00\u4e2a\u4f8b\u5b50:", "owner_org": null, "url_repo": null, "url_paper": null, "license": null, "languages": [], "difficulties": {"medium": 1000}, "categories": ["legal NLP benchmark"]}, "tasks": [{"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-0-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-0-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-1-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-1-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-10-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-10-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-11-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-11-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-12-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-12-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-13-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-13-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-14-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-14-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-15-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-15-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-16-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-16-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-17-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-17-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-18-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-18-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-19-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-19-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-2-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-2-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-20-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-20-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-21-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-21-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-22-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-22-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-23-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-23-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-24-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-24-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-3-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-3-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-4-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-4-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-5-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-5-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-6-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-6-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-7-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-7-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-8-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-8-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-9-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-1-9-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-0-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-0-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-1-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-1-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-10-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-10-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-11-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-11-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-12-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-12-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-13-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-13-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-14-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-14-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-15-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-15-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-16-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-16-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-17-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-17-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-18-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-18-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-19-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-19-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-2-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-2-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-20-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-20-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-21-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-21-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-22-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-22-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-23-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-23-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-24-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-24-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-3-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-3-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-4-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-4-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-5-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-5-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-6-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-6-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-7-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-7-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-8-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-8-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-9-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-1-2-9-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-0-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-0-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-1-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-1-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-10-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-10-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-11-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-11-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-12-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-12-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-13-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-13-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-14-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-14-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-15-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-15-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-16-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-16-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-17-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-17-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-18-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-18-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-19-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-19-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-2-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-2-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-20-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-20-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-21-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-21-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-22-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-22-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-23-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-23-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-24-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-24-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-3-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-3-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-4-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-4-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-5-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-5-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-6-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-6-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-7-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-7-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-8-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-8-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-9-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-1-9-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-0-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-0-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-1-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-1-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-10-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-10-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-11-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-11-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-12-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-12-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-13-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-13-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-14-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-14-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-15-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-15-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-16-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-16-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-17-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-17-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-18-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-18-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-19-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-19-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-2-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-2-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-20-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-20-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-21-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-21-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-22-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-22-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-23-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-23-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-24-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-24-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-3-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-3-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-4-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-4-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-5-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-5-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-6-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-6-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-7-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-7-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-8-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-8-zero-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-9-one-shot", "taskset": "lawbench"}, {"agent_timeout": 3600, "category": "legal NLP benchmark", "compose": false, "difficulty": "medium", "language": "", "oracle": null, "runnable": false, "tags": ["lawbench"], "task": "lawbench-2-10-9-zero-shot", "taskset": "lawbench"}], "next": "lawbench-2-10-9-zero-shot"}