{"harness": {"recipes": 0, "summary": null, "recipe": null, "tasks_passed": 3, "harness": "google-gemini-gemini-cli", "first_run": "2026-09-23T15:38:03", "domains": null, "runs": 6, "base_image": null, "finished": 6, "last_run": "2026-09-24T23:23:40", "repo": "https://github.com/google-gemini/gemini-cli", "passes": 3, "last_run_id": "20260924T232100-sw-gemini-cli-card_games__372", "use_case": null, "tasksets": ["aider_polyglot", "bird-bench", "humanevalfix", "swebench-verified"], "results": {"bird-bench/card_games__372": {"passes": 1, "last": "2026-09-24T23:23:40", "last_run": "20260924T232100-sw-gemini-cli-card_games__372", "last_tests": null, "last_outcome": "pass", "last_reward": 1, "runs": 1}, "aider_polyglot/polyglot_python_bowling": {"passes": 1, "last": "2026-09-24T15:44:28", "last_run": "20260924T154427-google-gemin-polyglot_python_bowling", "last_tests": {"summary": "31 passed", "total": 31, "passed": 31, "failed": 0, "agent_written": 0, "failed_names": []}, "last_outcome": "pass", "last_reward": 1, "runs": 2}, "aider_polyglot/polyglot_go_palindrome-products": {"passes": 0, "last": "2026-09-24T23:04:24", "last_run": "20260924T225606-sw-gemini-cli-polyglot_go_palindrome-products", "last_tests": null, "last_outcome": "fail", "last_reward": 0, "runs": 1}, "swebench-verified/psf__requests-5414": {"passes": 0, "last": "2026-09-24T23:12:23", "last_run": "20260924T230944-sw-gemini-cli-psf__requests-5414", "last_tests": {"summary": "1 failed, 130 passed, 1 xfailed, 158 errors", "total": 0, "passed": 0, "failed": 0, "agent_written": 0, "failed_names": []}, "last_outcome": "fail", "last_reward": 0, "runs": 1}, "humanevalfix/python-12": {"passes": 1, "last": "2026-09-24T23:20:36", "last_run": "20260924T231732-sw-gemini-cli-python-12", "last_tests": {"summary": "1 passed", "total": 1, "passed": 1, "failed": 0, "agent_written": 0, "failed_names": []}, "last_outcome": "pass", "last_reward": 1, "runs": 1}}, "models": ["bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"], "tasks_tried": 5, "api_style": "openai", "commit": "bedef96ef42905bd84a86dbec021c706168e7e2f"}, "profile": null, "recommendations": null, "runs": [{"run": "20260924T232100-sw-gemini-cli-card_games__372", "started": "2026-09-24T23:23:40", "finished": "2026-09-24T23:24:00", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "bird-bench", "name": "card_games__372"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 1, "verifier_rc": 0, "tests": null, "calls": 4, "seconds": 15, "input_tokens": 51212, "output_tokens": 399, "errors": 0, "last_action": "write_file: /app/answer.sql", "outcome": "scored", "verifier_says": "reward 1"}, {"run": "20260924T231732-sw-gemini-cli-python-12", "started": "2026-09-24T23:20:36", "finished": "2026-09-24T23:20:58", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "humanevalfix", "name": "python-12"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 1, "verifier_rc": 0, "tests": {"summary": "1 passed", "total": 1, "passed": 1, "failed": 0, "agent_written": 0, "failed_names": []}, "calls": 5, "seconds": 19, "input_tokens": 60399, "output_tokens": 858, "errors": 0, "last_action": "run_shell_command: cd /workspace && python Python__12.py", "outcome": "scored", "verifier_says": "reward 1 \u00b7 1 passed"}, {"run": "20260924T230944-sw-gemini-cli-psf__requests-5414", "started": "2026-09-24T23:12:23", "finished": "2026-09-24T23:17:30", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "swebench-verified", "name": "psf__requests-5414"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 0, "verifier_rc": 1, "tests": {"summary": "1 failed, 130 passed, 1 xfailed, 158 errors", "total": 0, "passed": 0, "failed": 0, "agent_written": 0, "failed_names": []}, "calls": 32, "seconds": 290, "input_tokens": 26811, "output_tokens": 500, "errors": 30, "last_action": "read_file: /testbed/tests/test_requests.py", "outcome": "scored", "verifier_says": "reward 0 \u00b7 1 failed, 130 passed, 1 xfailed, 158 errors \u00b7 verifier exited 1"}, {"run": "20260924T225606-sw-gemini-cli-polyglot_go_palindrome-products", "started": "2026-09-24T23:04:24", "finished": "2026-09-24T23:09:43", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "aider_polyglot", "name": "polyglot_go_palindrome-products"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 0, "verifier_rc": 0, "tests": null, "calls": 31, "seconds": 264, "input_tokens": 11709, "output_tokens": 125, "errors": 30, "last_action": "read_file: /app/go.mod", "outcome": "scored", "verifier_says": "reward 0"}, {"run": "20260924T154427-google-gemin-polyglot_python_bowling", "started": "2026-09-24T15:44:28", "finished": "2026-09-24T15:46:33", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "aider_polyglot", "name": "polyglot_python_bowling"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 1, "verifier_rc": 0, "tests": {"summary": "31 passed", "total": 31, "passed": 31, "failed": 0, "agent_written": 0, "failed_names": []}, "calls": 23, "seconds": 121, "input_tokens": 557543, "output_tokens": 16512, "errors": 0, "last_action": "run_shell_command: cd /app && python3 << 'EOF' from bowling import BowlingGame # Final comprehensive validation print(\"=\" * 50) print(\"FINA\u2026", "outcome": "scored", "verifier_says": "reward 1 \u00b7 31 passed"}, {"run": "20260923T153226-google-gemin-polyglot_python_bowling", "started": "2026-09-23T15:38:03", "finished": "2026-09-23T15:42:53", "status": "done", "kind": "harbor", "harness": "google-gemini-gemini-cli", "task": {"taskset": "aider_polyglot", "name": "polyglot_python_bowling"}, "model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", "reward": 0, "verifier_rc": 0, "tests": {"summary": "31 failed", "total": 31, "passed": 0, "failed": 31, "agent_written": 0, "failed_names": ["test_a_roll_cannot_score_more_than_10_points", "test_a_spare_followed_by_zeros_is_worth_ten_points", "test_a_spare_in_the_last_frame_gets_a_one_roll_bonus_that_is_counted_once", "test_a_strike_earns_ten_points_in_a_frame_with_a_single_roll", "test_a_strike_in_the_last_frame_gets_a_two_roll_bonus_that_is_counted_once", "test_a_strike_with_the_one_roll_bonus_after_a_spare_in_the_last_frame_does_not_get_a_bonus", "test_all_strikes_is_a_perfect_game", "test_an_incomplete_game_cannot_be_scored", "test_an_unstarted_game_cannot_be_scored", "test_bonus_roll_after_a_strike_in_the_last_frame_cannot_score_more_than_10_points", "test_bonus_roll_for_a_spare_in_the_last_frame_must_be_rolled_before_score_can_be_calculated", "test_bonus_rolls_for_a_strike_in_the_last_frame_must_be_rolled_before_score_can_be_calculated", "test_both_bonus_rolls_for_a_strike_in_the_last_frame_must_be_rolled_before_score_can_be_calculated", "test_cannot_roll_after_bonus_roll_for_spare", "test_cannot_roll_after_bonus_rolls_for_strike", "test_cannot_roll_if_game_already_has_ten_frames", "test_consecutive_spares_each_get_a_one_roll_bonus", "test_consecutive_strikes_each_get_the_two_roll_bonus", "test_last_two_strikes_followed_by_only_last_bonus_with_non_strike_points", "test_points_scored_in_the_roll_after_a_spare_are_counted_twice", "test_points_scored_in_the_two_rolls_after_a_strike_are_counted_twice_as_a_bonus", "test_rolling_a_spare_with_the_two_roll_bonus_does_not_get_a_bonus_roll", "test_rolls_cannot_score_negative_points", "test_second_bonus_roll_after_a_strike_in_the_last_frame_cannot_score_more_than_10_points", "test_should_be_able_to_score_a_game_with_all_zeros", "test_should_be_able_to_score_a_game_with_no_strikes_or_spares", "test_strikes_with_the_two_roll_bonus_do_not_get_bonus_rolls", "test_the_second_bonus_rolls_after_a_strike_in_the_last_frame_cannot_be_a_strike_if_the_first_one_is_not_a_strike", "test_two_bonus_rolls_after_a_strike_in_the_last_frame_can_score_more_than_10_points_if_one_is_a_strike", "test_two_bonus_rolls_after_a_strike_in_the_last_frame_cannot_score_more_than_10_points", "test_two_rolls_in_a_frame_cannot_score_more_than_10_points"]}, "calls": 30, "seconds": 286, "input_tokens": 0, "output_tokens": 0, "errors": 30, "last_action": null, "outcome": "scored", "verifier_says": "reward 0 \u00b7 31 failed"}]}