{"taskset": {"n_runnable": 0, "name": null, "url_data": null, "catalog_id": null, "owner_type": null, "cited_by": null, "taskset": "research-code-bench", "domain": "swe", "task_kind": null, "environment": null, "grading": null, "path": "datasets/research-code-bench", "n_single_container": 212, "n_tasks": 212, "domain_raw": null, "sample_instruction": "You are an expert in reproducing research code from a paper.\n\n## Paper Content\nHere is the paper that you need to use to complete the code:\n<paper>\n\n\\documentclass{article} \\usepackage{iclr2025_conference,times}\n\n\n\n\\usepackage{amsmath,amsfonts,bm}\n\n\\newcommand{\\figleft}{{\\em (Left)}}\n\\newcommand{\\figcenter}{{\\em (Center)}}\n\\newcommand{\\figright}{{\\em (Right)}}\n\\newcommand{\\figtop}{{\\em (Top)}}\n\\newcommand{\\figbottom}{{\\em (Bottom)}}\n\\newcommand{\\captiona}{{\\em (a)}}\n\\newcommand{\\captionb}{{\\em (b)}}\n\\newcommand{\\captionc}{{\\em (c)}}\n\\newcommand{\\captiond}{{\\em (d)}}\n\n\\newcommand{\\newterm}[1]{{\\bf #1}}\n\n\n\\def\\figref#1{figure~\\ref{#1}}\n\\def\\Figref#1{Figure~\\ref{#1}}\n\\def\\twofigref#1#2{figures \\ref{#1} and \\ref{#2}}\n\\def\\quadfigref#1#2#3#4{figures \\ref{#1}, \\ref{#2}, \\ref{#3} and \\ref{#4}}\n\\def\\secref#1{section~\\ref{#1}}\n\\def\\Secref#1{Section~\\ref{#1}}\n\\def\\twosecrefs#1#2{sections \\ref{#1} and \\ref{#2}}\n\\def\\secrefs#1#2#3{sections \\ref{#1}, \\ref{#2} and \\ref{#3}}\n\\def\\eqref#1{equation~\\ref{#1}}\n\\def\\Eqref#1{Equation~\\ref{#1}}\n\\def\\plaineqref#1{\\ref{#1}}\n\\def\\chapref#1{chapter~\\ref{#1}}\n\\def\\Chapref#1{Chapter~\\ref{#1}}\n\\def\\rangechapref#1#2{chapters\\ref{#1}--\\ref{#2}}\n\\def\\algref#1{algorithm~\\ref{#1}}\n\\def\\Algref#1{Algorithm~\\ref{#1}}\n\\def\\twoalgref#1#2{algorithms \\ref{#1} and \\ref{#2}}\n\\def\\Twoalgref#1#2{Algorithms \\ref{#1} and \\ref{#2}}\n\\def\\partref#1{part~\\ref{#1}}\n\\def\\Partref#1{Part~\\ref{#1}}\n\\def\\twopartref#1#2{parts \\ref{#1} and \\ref{#2}}\n\n\\def\\ceil#1{\\lceil #1 \\rceil}\n\\de", "owner_org": null, "url_repo": null, "url_paper": null, "license": null, "languages": [], "difficulties": {"difficult": 212}, "categories": ["code-generation"]}, "tasks": [{"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "advantage-alignment"], "task": "advantage-alignment_aa_terms1", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "advantage-alignment"], "task": "advantage-alignment_aa_terms2", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "advantage-alignment"], "task": "advantage-alignment_aa_terms3", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "advantage-alignment"], "task": "advantage-alignment_integrated_aa", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "advantage-alignment"], "task": "advantage-alignment_proximal_surrogate", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_empirical_lambda_init_function", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_groupnorm_with_scaling", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_lambda_setup", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_multihead_differential_attention", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_normalization_setup", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_re-parameterize_lambda", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "Diff-Transformer"], "task": "diff-transformer_taking_difference", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_calculate_model_losses", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_calculate_reference_losses", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_dpo_loss_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_dpo_loss_formula", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_implicit_accuracy_metric", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_loss_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_raw_model_loss_metric", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_raw_ref_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2024", "DiffusionDPO"], "task": "diffusiondpo_sft_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_alpha", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_bias", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_channel_split", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_channels_first", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_channels_last", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_forward", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_init", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_tanh", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "DyT"], "task": "dyt_weight", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_activation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_add_query_tokens", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_apply_masking_probability", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_calculate_upscale_parameters", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_convolution_1", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_convolution_2", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_create_attention_mask", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_custom_attention_mechanism", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_disable_attention_mask", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_eomt_initialization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_extract_query_key_value", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_extract_query_tokens", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_final_prediction", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_forward_pass", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_generate_class_logits", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_generate_mask_logits", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_generate_masks", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_initialize_class_prediction_head", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_initialize_mask_prediction_head", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_initialize_query_embedding", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_initialize_tracking_variables", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_initialize_upscale_module", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_input_normalization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_normalization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_patch_embedding", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_predict_from_features", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_predict_masks_and_classes", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_process_attention_mask", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_process_image_features", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_process_transformer_blocks", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_project_attention_output", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_random_mask_disable", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_register_attention_mask_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_scale_block_forward", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_scale_block_initialization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_store_parameters", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "cvpr2025", "eomt"], "task": "eomt_transformer_block_forward", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_cfg_schedule", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_chunk_mask_to_pred", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_concatenate_patches", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_init_token_mask__patches__orders__num_iter", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_mask_by_order", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_mask_ratio_and_mask_length", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_next_level_sample_function", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_patchify", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_random_masking", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_random_masking_uniform", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_sample_image", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_sample_orders", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "fractalgen"], "task": "fractalgen_unpatchify", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_apply_2nd_order_gm", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_apply_mask_to_gm", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_apply_probabilistic_cfg", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_final_nll", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_gaussian_log_likelihood", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_gaussian_mask", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_inverse_stds", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_noise_scale", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_timestep_coefficients", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_compute_weighted_difference", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_convert_model_output_to_x0", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_gm_kl_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_main_sampling_loop", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_perform_gm_sdeode_step", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_prepare_timestep_tensor", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "GMFlow"], "task": "gmflow_set_timesteps", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_chamfer_distance", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_distance_between_two_sets", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_euclidean_distance_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_gps_reparameterization_with_minimum_euclidean_distance", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_gumbel_distribution_similarity_transformation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "GPS"], "task": "gps_pairwise_expansion", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry__dx_to_theta_id_dr", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_activity_metrics", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_apply_additive_transformation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_apply_nonlinear_transform", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_apply_transformation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_calculate_distance_preservation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_compute_neural_representations", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_compute_transformation_error", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_compute_transformation_matrix", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_conformal_isometry_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_linear_transformation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_nonlinear_transformation_1", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_prepare_displacement", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_prepare_inputs", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "grid-cell-conformal-isometry"], "task": "grid-cell-conformal-isometry_transformation_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "hyla"], "task": "hyla_mlp_linear", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "hyla"], "task": "hyla_mlp_relu", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "hyla"], "task": "hyla_rms_head", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_combine_gradients", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_compute_gradients", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_compute_schur_decomposition", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_construct_jacobian_matrix", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_gamma_update_function", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_gradient_descent_ascent_field", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_initialize_variables", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_jacobian_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_lazy_extra_newton", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_main_iteration_loop", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_split_input_and_compute_norm", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_split_input_vector", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_track_metrics", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "LEN"], "task": "len_update_steps", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_ai_text_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_alpha_estimation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_alpha_optimization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_bootstrap_inference", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_bootstrap_sampling", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_confidence_interval_calculation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_data_loading", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_data_preprocessing", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_human_text_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_inference_pipeline", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_loading_token_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_log_likelihood_optimization", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_mixture_log_likelihood_calculation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_precompute_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "colm2024", "llm-sci-use"], "task": "llm-sci-use_token_distribution_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_apply_filter_to_scores", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_convert_logits_to_probabilities", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_ensure_minimum_tokens_are_kept", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_find_maximum_probability_token", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_identify_tokens_to_remove", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_min-p_sampling", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "minp"], "task": "minp_scale_min_p_threshold", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_1", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_2", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_3", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_4", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_for_each_sample", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_initialization_for_each_sample", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_search", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_dynamic_programming_traceback", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_search_oss_function", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_teacher_trajectory_computation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "OptimalSteps"], "task": "optimalsteps_teacher_trajectory_computation_for_each_timestep", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "REPA-E"], "task": "repa-e_check1", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "REPA-E"], "task": "repa-e_check2", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "REPA-E"], "task": "repa-e_sit_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "REPA-E"], "task": "repa-e_sitbatchnorm_eps_is_set_to_1e-4", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "REPA-E"], "task": "repa-e_vae_loss", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_averaging_update", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_averaging_update_and_interpolation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_bias_correction", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_decay_the_first_and_second_moment_running_average_coefficient", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_denom", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_dynamic_weighting", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_dynamic_weighting_and_averaging", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_gradient_evaluation_interpolation", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_learning_rate", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_moment_running_average_coefficient", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_step", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_warmup_schedule", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_weight_decay", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "neurips2024", "schedule_free"], "task": "schedule_free_z_update", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_apply_embedding_masks", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_decode_multiple_tokens", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_decode_single_token", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_generate_main_function", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_generate_remaining_tokens", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_get_cfg_scale", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_initialize_sequence", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_prefill_initial_tokens", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_prefill_sequence", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_setup_conditions", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "arxiv2025", "semanticist"], "task": "semanticist_setup_sequence_parameters", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "SISS"], "task": "siss_compute_loss_terms", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "SISS"], "task": "siss_create_defensive_mixture", "taskset": "research-code-bench"}, {"agent_timeout": 3600, "category": "code-generation", "compose": false, "difficulty": "difficult", "language": "", "oracle": null, "runnable": false, "tags": ["research-code-bench", "paper2code", "iclr2025", "SISS"], "task": "siss_double_forward_passes", "taskset": "research-code-bench"}], "next": "siss_double_forward_passes"}