{
  "schema_version": 1,
  "source_id": "bankertoolbench",
  "repository": "https://github.com/Handshake-AI-Research/bankertoolbench",
  "revision": "c3816821fed0c85b975f15ffb03af85a63448012",
  "dataset_repository": "https://huggingface.co/datasets/handshake-ai-research/bankertoolbench",
  "dataset_revision": "6407cdc22a218d59a1e41d8598679d02d821d6ff",
  "dataset_sha256": "83d8e66fd9f4106bff12a0efdccd3965b581a7626ef5ec0c50ed8609931fea49",
  "review_binding": {
    "path": "candidates/bankertoolbench.json",
    "sha256": "e5ab0c27eb0ba6490cacf948a3beb2f2b89abd1eac0ffbf0cd19929ddc31726b",
    "canonical_sha256": "39b2917bfb0e75cd9a6ca80ac45c8ddb2be9cf41952c0699f49cd0dfdabba875"
  },
  "implementation_mirrored": false,
  "review_scope": "static_source_configuration_not_observed_runtime_failure",
  "components": {
    "gandalf": {
      "repository": "https://github.com/Handshake-AI-Research/gandalf-the-grader",
      "revision": "b30a864da4f8fce7f43f4577228dd5c9ed22b3d7",
      "version": "1.0.0",
      "identical_wheel_sdist_tag_files": 8
    },
    "harbor": {
      "version": "0.15.0",
      "identity": "Exact PyPI wheel URL and SHA-256; no Git revision inferred."
    }
  },
  "entrypoints": [
    {
      "id": "adapter",
      "component": "code",
      "path": "adapters/btb/adapter.py",
      "source_url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/adapter.py#L60-L265",
      "bytes": 11482,
      "sha256": "86c10f81c0ff8b099dddd417e08b889c4b6e1e40cecb98b1fd0f2f0e988d8f06",
      "line_range": [
        60,
        265
      ]
    },
    {
      "id": "test-sh",
      "component": "code",
      "path": "adapters/btb/template/tests/test.sh",
      "source_url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/template/tests/test.sh",
      "bytes": 1692,
      "sha256": "6b7a07caeddfe61647b5ecf23c97aef8f748313f0851537fb23edc704bb0d12e"
    },
    {
      "id": "g-orchestrator",
      "component": "gandalf",
      "path": "src/gandalf/orchestrator.py",
      "source_url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/orchestrator.py#L244-L746",
      "bytes": 26853,
      "sha256": "7c2c052b4ca0d04ba952432d4facf38a335128c6bd95cbe67a3173a8a82aef80",
      "line_range": [
        244,
        746
      ]
    },
    {
      "id": "g-parser",
      "component": "gandalf",
      "path": "src/gandalf/judge.py",
      "source_url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/judge.py#L92-L166",
      "bytes": 11663,
      "sha256": "176f65c3e4c9edbfb9b4eea876e5b63216e2473ddbf9e00d7cb76706e73a9df7",
      "line_range": [
        92,
        166
      ]
    },
    {
      "id": "g-models",
      "component": "gandalf",
      "path": "src/gandalf/models.py",
      "source_url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/models.py#L150-L165",
      "bytes": 7184,
      "sha256": "659241248c3d16d9f11e2eb8c9e67c43bd3261f5db7c91af59cca626655f6fac",
      "line_range": [
        150,
        165
      ]
    }
  ],
  "distributions": [
    {
      "package": "gandalf-the-grader",
      "version": "1.0.0",
      "filename": "gandalf_the_grader-1.0.0-py3-none-any.whl",
      "url": "https://files.pythonhosted.org/packages/e7/d3/3af8f410ccea8525edd814d0de8609f34f544bff8c61e62a623a7d839d06/gandalf_the_grader-1.0.0-py3-none-any.whl",
      "bytes": 31337,
      "sha256": "486cff4222cca70c2637b3c62e306edebeedbdd467108ec27c28e77003f620c7",
      "downloaded_and_digest_verified": true,
      "upload_time": "2026-04-13T03:43:20.084133Z"
    },
    {
      "package": "gandalf-the-grader",
      "version": "1.0.0",
      "filename": "gandalf_the_grader-1.0.0.tar.gz",
      "url": "https://files.pythonhosted.org/packages/aa/e0/a21a5e847df2b506ac679d6c0a7c26493e4b2bc41c2ee9990b9deb62a537/gandalf_the_grader-1.0.0.tar.gz",
      "bytes": 326525,
      "sha256": "1906c746aabcdc9663a94046c74be6971568a8c829a228a290c89020bac40ac7",
      "downloaded_and_digest_verified": true,
      "upload_time": "2026-04-13T03:43:23.589107Z"
    },
    {
      "package": "harbor",
      "version": "0.15.0",
      "filename": "harbor-0.15.0-py3-none-any.whl",
      "url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "bytes": 1463875,
      "sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b",
      "downloaded_and_digest_verified": true,
      "upload_time": "2026-06-19T22:48:19.009526Z"
    },
    {
      "package": "harbor",
      "version": "0.15.0",
      "filename": "harbor-0.15.0.tar.gz",
      "url": "https://files.pythonhosted.org/packages/e0/af/9927af345cd7d80e4346539e2340a1cd795861b12fb5d5ffc7b0788a7401/harbor-0.15.0.tar.gz",
      "bytes": 1287081,
      "sha256": "406b20ce520fc1ec12bc9b05396cc91756bb6379fb565a5bc2f2b6759e94cb12",
      "downloaded_and_digest_verified": false,
      "upload_time": "2026-06-19T22:48:17.474870Z"
    }
  ],
  "evidence": [
    {
      "id": "dataset",
      "component": "dataset",
      "path": "tasks.jsonl",
      "sha256": "83d8e66fd9f4106bff12a0efdccd3965b581a7626ef5ec0c50ed8609931fea49",
      "bytes": 3041602,
      "url": "https://huggingface.co/datasets/handshake-ai-research/bankertoolbench/blob/6407cdc22a218d59a1e41d8598679d02d821d6ff/tasks.jsonl"
    },
    {
      "id": "data-card",
      "component": "dataset",
      "path": "README.md",
      "sha256": "1cc56483ff655104de3fd892d123229cea9e18eb049febc2c2f6e6e28fc7406f",
      "bytes": 5332,
      "url": "https://huggingface.co/datasets/handshake-ai-research/bankertoolbench/blob/6407cdc22a218d59a1e41d8598679d02d821d6ff/README.md"
    },
    {
      "id": "changelog",
      "component": "dataset",
      "path": "CHANGELOG.md",
      "sha256": "65e5fbccd5b88c734e3ca3a96f1244865f6e7fb85fd01e54aef7c28742f51e18",
      "bytes": 642,
      "url": "https://huggingface.co/datasets/handshake-ai-research/bankertoolbench/blob/6407cdc22a218d59a1e41d8598679d02d821d6ff/CHANGELOG.md"
    },
    {
      "id": "golden-readme",
      "component": "dataset",
      "path": "golden-outputs/README.md",
      "sha256": "8876134894dcb725df126f7067a27f5369101926bd39e42910aa57aa893b4ce4",
      "bytes": 251,
      "url": "https://huggingface.co/datasets/handshake-ai-research/bankertoolbench/blob/6407cdc22a218d59a1e41d8598679d02d821d6ff/golden-outputs/README.md"
    },
    {
      "id": "code-readme",
      "component": "code",
      "path": "README.md",
      "sha256": "27cc8a15ca0b68452443dd85dae6b1f82df7d57bc8d367a30bf6930b19e06a1c",
      "bytes": 15138,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/README.md"
    },
    {
      "id": "code-license",
      "component": "code",
      "path": "LICENSE",
      "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
      "bytes": 11358,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/LICENSE"
    },
    {
      "id": "schema",
      "component": "code",
      "path": "adapters/btb/schema.py",
      "sha256": "4078dbbf574eb838dce32253d79fe477fa2b242d786441ea150e6fdd1808b317",
      "bytes": 4778,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/schema.py#L12-L142",
      "line_range": [
        12,
        142
      ]
    },
    {
      "id": "cli",
      "component": "code",
      "path": "adapters/btb/run_adapter.py",
      "sha256": "cdb06403823b10447a7f73bd8f1aeb3dfa4874db6805aa3db056d94f10bbe00f",
      "bytes": 7718,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/run_adapter.py#L57-L64",
      "line_range": [
        57,
        64
      ]
    },
    {
      "id": "adapter",
      "component": "code",
      "path": "adapters/btb/adapter.py",
      "sha256": "86c10f81c0ff8b099dddd417e08b889c4b6e1e40cecb98b1fd0f2f0e988d8f06",
      "bytes": 11482,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/adapter.py#L60-L265",
      "line_range": [
        60,
        265
      ]
    },
    {
      "id": "test-sh",
      "component": "code",
      "path": "adapters/btb/template/tests/test.sh",
      "sha256": "6b7a07caeddfe61647b5ecf23c97aef8f748313f0851537fb23edc704bb0d12e",
      "bytes": 1692,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/template/tests/test.sh"
    },
    {
      "id": "dockerfile",
      "component": "code",
      "path": "adapters/btb/template/environment/Dockerfile",
      "sha256": "9f169acb08f8a52bde5b372a2c5fcd6bada7fbc8de288e8ea54e0d8166cddfc2",
      "bytes": 5187,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/template/environment/Dockerfile#L1-L109",
      "line_range": [
        1,
        109
      ]
    },
    {
      "id": "runtime-requirements",
      "component": "code",
      "path": "adapters/btb/template/environment/agent-python-requirements.txt",
      "sha256": "2876b3002dedf35ed5588e5614f959a63177a0c9e2c30c39604e96591390aac3",
      "bytes": 224,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/adapters/btb/template/environment/agent-python-requirements.txt"
    },
    {
      "id": "job",
      "component": "code",
      "path": "job.yaml",
      "sha256": "60029ecbae22cf63fd274ed424b5baa63941226e29c67752a23aa909fa595580",
      "bytes": 465,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/job.yaml"
    },
    {
      "id": "system-prompt",
      "component": "code",
      "path": "prompts/system_prompt.j2",
      "sha256": "c4cfd3ab1dac520605dac6bc4b26d45414b33cdb912b78e859e16d368d3ee693",
      "bytes": 8020,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/prompts/system_prompt.j2"
    },
    {
      "id": "replay",
      "component": "code",
      "path": "verifier-eval/verifier_runner/rollout.py",
      "sha256": "454c62e1bb82d80487cff580b01af6e2c27aaf0ba65729afcee51486ce442fb0",
      "bytes": 26709,
      "url": "https://github.com/Handshake-AI-Research/bankertoolbench/blob/c3816821fed0c85b975f15ffb03af85a63448012/verifier-eval/verifier_runner/rollout.py#L454-L461",
      "line_range": [
        454,
        461
      ]
    },
    {
      "id": "g-models",
      "component": "gandalf",
      "path": "src/gandalf/models.py",
      "sha256": "659241248c3d16d9f11e2eb8c9e67c43bd3261f5db7c91af59cca626655f6fac",
      "bytes": 7184,
      "url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/models.py#L150-L165",
      "line_range": [
        150,
        165
      ]
    },
    {
      "id": "g-parser",
      "component": "gandalf",
      "path": "src/gandalf/judge.py",
      "sha256": "176f65c3e4c9edbfb9b4eea876e5b63216e2473ddbf9e00d7cb76706e73a9df7",
      "bytes": 11663,
      "url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/judge.py#L92-L166",
      "line_range": [
        92,
        166
      ]
    },
    {
      "id": "g-orchestrator",
      "component": "gandalf",
      "path": "src/gandalf/orchestrator.py",
      "sha256": "7c2c052b4ca0d04ba952432d4facf38a335128c6bd95cbe67a3173a8a82aef80",
      "bytes": 26853,
      "url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/src/gandalf/orchestrator.py#L244-L746",
      "line_range": [
        244,
        746
      ]
    },
    {
      "id": "g-dependencies",
      "component": "gandalf",
      "path": "pyproject.toml",
      "sha256": "df19ba85fdb8696293dd013ee470c4692107634d1bc357df466f68ffdf9cedcc",
      "bytes": 2359,
      "url": "https://github.com/Handshake-AI-Research/gandalf-the-grader/blob/b30a864da4f8fce7f43f4577228dd5c9ed22b3d7/pyproject.toml"
    },
    {
      "id": "h-base",
      "component": "harbor",
      "path": "harbor/metrics/base.py",
      "sha256": "373de596294fde446c1b2d73cd7af7f35852407b63cb852def14c63584c11236",
      "bytes": 928,
      "distribution_url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "distribution_sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b",
      "line_range": [
        14,
        35
      ]
    },
    {
      "id": "h-mean",
      "component": "harbor",
      "path": "harbor/metrics/mean.py",
      "sha256": "fcebbfd69bb2b52c52da59af9ca4ef6080bd7064d8c3a301a9762353ed71efbc",
      "bytes": 396,
      "distribution_url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "distribution_sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b"
    },
    {
      "id": "h-job",
      "component": "harbor",
      "path": "harbor/job.py",
      "sha256": "1d8e10326e1011a0bc65947b6bb9087e69e6c219fb2a17ec3784bc1e75e7ca2c",
      "bytes": 36311,
      "distribution_url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "distribution_sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b",
      "line_range": [
        762,
        796
      ]
    },
    {
      "id": "h-config",
      "component": "harbor",
      "path": "harbor/models/job/config.py",
      "sha256": "6a7f64372169895d2ff81bc260d8674d1d542d3a997f9c7d1ab9af358eb0e654",
      "bytes": 14466,
      "distribution_url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "distribution_sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b",
      "line_range": [
        285,
        330
      ]
    },
    {
      "id": "h-verifier",
      "component": "harbor",
      "path": "harbor/verifier/verifier.py",
      "sha256": "7110e7b854ec216f9bbef2e3b03712f2fd2614d5d0dc8a9a1768c0e887e129fc",
      "bytes": 8659,
      "distribution_url": "https://files.pythonhosted.org/packages/df/5a/6e860d3bdbb09c56296d4b57fcebc22f8eb014b06eb3810f9e0a8ab718f7/harbor-0.15.0-py3-none-any.whl",
      "distribution_sha256": "a6f7f703d9b008920f07751ae27f6c049afe908bee8199aa6bcc911f6c2aab1b",
      "line_range": [
        137,
        238
      ]
    }
  ],
  "verifier_contract": {
    "binding": "Each task aggregated_rubric_json becomes tests/rubric.json; effective instructions, deliverables directory and trajectory are passed through grader.toml to Gandalf 1.0.0.",
    "kind": "llm_artifact_rubric_judge",
    "status": "source_configuration_and_implementation_statically_reviewed_not_run",
    "model_default": "gemini/gemini-3-flash-preview",
    "batch_splits": 2,
    "max_concurrency": 2,
    "judge_retries": 1,
    "judge_timeout_seconds": 300,
    "batch_timeout_seconds": 1740,
    "outer_verifier_timeout_seconds": 1800,
    "agent_timeout_seconds": 3000,
    "score": "round(clip(sum(weight where met is True) / sum(all positive rubric weights), 0, 1), 4); actual BTB weights are all positive.",
    "coverage": "evaluated_criteria_pct is item-count coverage, not weighted coverage.",
    "missing_and_failed": "Missing/invalid indexed verdicts become met=None; only unresolved criteria are retried once. Remaining handled errors produce diagnostic info.json and exit 1 without a new reward.json.",
    "empty_deliverables": "BTB test.sh returns reward 0 and empty criteria without invoking Gandalf if the deliverables directory is absent/empty.",
    "outer_aggregation": "Harbor 0.15.0 default mean maps missing rewards to 0 and divides by the number of actual TrialResult entries in each agent/model/dataset group. Default n_attempts=1 and outer max_retries=0; BTB job.yaml does not override either.",
    "formal_vs_replay": "Harbor reads reward files, not info.reward. BTB verifier replay code can fall back to info.reward for diagnostics.",
    "evidence_ids": [
      "adapter",
      "test-sh",
      "g-models",
      "g-parser",
      "g-orchestrator",
      "h-base",
      "h-mean",
      "h-job",
      "h-config",
      "h-verifier",
      "replay"
    ]
  },
  "environment_contract": {
    "type": "Linux Docker artifact workspace with terminal/programming and local custom MCP services",
    "reference_environment_declared": {
      "Harbor": "0.15.0",
      "OpenCode": "1.17.9",
      "Docker_Engine": "29.1.5",
      "uv": "0.10.12",
      "Python_container_runtime": "3.12"
    },
    "container_template": "Ubuntu 24.04 tag; agent Python 3.14; OpenCode and uv use latest in the source template. No built image digest was verified.",
    "network_policy": "No-external-internet instruction coexists with allow_internet=True; isolation was not tested.",
    "required_external_access_for_evaluation": "LLM provider credentials/model access and environment dependencies; none supplied or used here.",
    "interface_limit": "The custom market-data and SEC tool services are benchmark software, not proof of access to commercial FactSet/Capital IQ products or a GUI-only workflow.",
    "gandalf_dependencies": {
      "base_requires_dist": [
        "jinja2<4,>=3.1",
        "litellm<2,>=1.82.0",
        "openhands-sdk<1.16.1,>=1.11.2",
        "openhands-tools<1.16.1,>=1.11.2",
        "pydantic<3,>=2.12.5"
      ],
      "pinned_requirement_entries": 439,
      "unique_pinned_dependency_names": 339,
      "key_pins": {
        "jinja2": "3.1.6",
        "litellm": "1.83.0",
        "openhands-sdk": "1.14.0",
        "openhands-tools": "1.16.0",
        "pydantic": "2.12.5",
        "openai": "2.31.0",
        "anthropic": "0.94.0",
        "tenacity": "9.1.4"
      },
      "post_install_requirement_differences": {
        "pypdf": {
          "gandalf_extra": "6.10.0",
          "btb_runtime": "6.7.1"
        },
        "pillow": {
          "gandalf_extra": "12.2.0",
          "btb_runtime": "12.1.1"
        }
      },
      "limit": "Requirements were statically read. No resolver run, full transitive audit, SDK transport-retry audit or installed-environment compatibility claim."
    }
  },
  "review_limits": [
    {
      "id": "BTB-VERSION-01",
      "finding": "Current and v1.0 have the same 100 UUIDs, but final_prompt, both optional contexts and aggregated_rubric_json changed in every row; rubric items fell from 15,054 to 10,967.",
      "implication": "Import at most 100 current definition rows; attach the current pin and never assign paper v1.0 scores to v1.1.",
      "evidence_ids": [
        "dataset",
        "changelog",
        "code-readme"
      ],
      "scope": "exact structural comparison"
    },
    {
      "id": "BTB-INSTRUCTION-01",
      "finding": "CLI flags default both contexts off; direct adapter/schema helpers default them on. Runtime strips text; two raw final_prompt values change under strip().",
      "implication": "Preserve all nine raw fields. State final_prompt as the selected baseline and record flags rather than silently appending contexts or rewriting raw text.",
      "evidence_ids": [
        "schema",
        "cli",
        "adapter"
      ],
      "affected_raw_trim_ids": [
        "4a3bf36b-caeb-4825-a60c-4c416b4d555c",
        "92235649-0d63-4396-b864-a64257e16746"
      ],
      "scope": "static protocol"
    },
    {
      "id": "BTB-VERDICT-01",
      "finding": "Verdict.from_raw coerces non-null met with Python bool(); a nonempty string such as \"false\" is truthy. Batch parsing ignores invalid/out-of-range indexes, keeps the last duplicate index, and marks missing indexes unresolved.",
      "implication": "This is a static validation boundary, not an observed misgrade. Future runs need typed verdict validation and error coverage retained alongside the score.",
      "evidence_ids": [
        "g-models",
        "g-parser"
      ],
      "scope": "exact released source, no judge calls"
    },
    {
      "id": "BTB-ERROR-01",
      "finding": "One retry applies only to met=None criteria. After handled judge errors remain, info.json retains a diagnostic score with the full positive-weight denominator, but the process exits 1 without writing a new reward.json. The replay helper can fall back to info.reward.",
      "implication": "Keep diagnostic partial scores, successful rewards and grader errors distinct. A reused output directory can retain an older reward because this entrypoint does not remove it; no such runtime event was observed.",
      "evidence_ids": [
        "g-orchestrator",
        "replay"
      ],
      "scope": "static outcome paths; initialization/schema exceptions may occur before info writing"
    },
    {
      "id": "BTB-OUTER-MEAN-01",
      "finding": "Harbor 0.15.0 default Mean inserts zero for a missing reward; its denominator is the actual TrialResult list for the agent/model/dataset, including error entries. Default outer retries=0 and n_attempts=1.",
      "implication": "Complete 100-task single-attempt jobs have a 100-entry mean; partial/cancelled jobs require the actual result count. Do not describe it as valid-result-only averaging or infer all requested tasks completed.",
      "evidence_ids": [
        "h-base",
        "h-mean",
        "h-job",
        "h-config",
        "h-verifier",
        "job"
      ],
      "scope": "exact wheel source, selected final aggregation path"
    },
    {
      "id": "BTB-TIMEOUT-01",
      "finding": "The BTB defaults produce a 1,740-second batch cap inside an 1,800-second verifier. An initial batch plus the allowed retry can exceed the outer budget.",
      "implication": "One configured retry is not a guarantee that the full retry budget can finish; timeout frequency was not measured.",
      "evidence_ids": [
        "adapter",
        "g-orchestrator"
      ],
      "scope": "conditional budget arithmetic"
    },
    {
      "id": "BTB-ENVIRONMENT-01",
      "finding": "README names Harbor 0.15.0 and OpenCode 1.17.9, but Dockerfile still uses opencode-ai@latest, uv:latest and unpinned apt packages. Later runtime installation specifies pypdf 6.7.1 and Pillow 12.1.1 versus Gandalf pinned-extra 6.10.0/12.2.0.",
      "implication": "The grader distribution is pinned; the complete environment is not proven identical to the published reference environment. No installation, dependency resolution or build was performed.",
      "evidence_ids": [
        "code-readme",
        "dockerfile",
        "runtime-requirements",
        "g-dependencies"
      ],
      "scope": "declared requirements comparison"
    },
    {
      "id": "BTB-NETWORK-01",
      "finding": "Source prompts prohibit external internet use, but generated task.toml sets allow_internet=True.",
      "implication": "Treat this as an instruction policy, not proof of network isolation. Local custom market/SEC MCP services do not establish use of a named commercial data terminal or a GUI-only interface.",
      "evidence_ids": [
        "adapter",
        "system-prompt"
      ],
      "scope": "static configuration"
    },
    {
      "id": "BTB-RUBRIC-01",
      "finding": "Current rubrics average 109.67 items; two tasks have no Risk & Compliance category. Categories are retained by the adapter but not used as score weights by Gandalf RubricItem.",
      "implication": "Report exact current criterion counts and weighted scoring; do not assert every task covers all six categories or that ~150 is the current mean.",
      "affected_task_ids": [
        "aa577c1d-2c22-4963-91c4-456ac32e5b4a",
        "4fedc1aa-5e19-4339-9f0b-f455239b2655"
      ],
      "evidence_ids": [
        "dataset",
        "schema",
        "g-models"
      ],
      "scope": "structural counts; not a rubric-truth defect claim"
    },
    {
      "id": "BTB-LICENSE-01",
      "finding": "The dataset card declares CC-BY-4.0 and noai; each original row carries the publisher no-training canary. The code and exact Gandalf release carry Apache-2.0 licenses.",
      "implication": "Definition collection should retain attribution, canary and stated no-training notice. Data-license declarations do not independently verify rights or privacy of every external binary asset.",
      "evidence_ids": [
        "data-card",
        "code-license",
        "dataset"
      ],
      "scope": "publisher declarations; no legal certification or asset-content review"
    }
  ]
}