{
  "schema_version": 1,
  "repository": "https://github.com/FinWorkBench/Finch",
  "revision": "95a8b8d135a528b325be003e54c55f886a22602d",
  "dataset_repository": "https://huggingface.co/datasets/FinWorkBench/Finch",
  "dataset_revision": "530700c28f7eb68a66c7753d7857c5be80ca274d",
  "dataset_sha256": "0c8379da255b46c8e5022643d54fe4d448cde3ce8406e964744a9be651ba7c1f",
  "license_observation": "No code license declaration found in the full pinned main tree; dataset CC-BY declaration does not license the separate implementation repository.",
  "storage_policy": "Metadata and immutable URLs only; implementation source is not mirrored or executed.",
  "implementation_mirrored": false,
  "review_scope": "Static source inspection; current main evaluator is not a reproduction of original-paper runs. All limits below describe evidence coverage, not observed task failures.",
  "files": {
    "README.md": {
      "bytes": 10074,
      "sha256": "c0988d6c61c909633859efcf95463550bfeb62319d45fd7b8ab1742746e610ca",
      "git_blob_oid": "7adcd61a15781ceb92264aeefe414d94ab5e68c7",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/README.md"
    },
    "src/build_prompt/content_builder/config.py": {
      "bytes": 3417,
      "sha256": "e70b601507a4ef188094f61d62204b2b63652b44ef4df54562b8cc99f8f54e74",
      "git_blob_oid": "f3775ee25f9092fa922f7bf54c4eff57ab9aab01",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/config.py"
    },
    "src/build_prompt/content_builder/content_builder.py": {
      "bytes": 19769,
      "sha256": "69122ea7318818883c0628fbe583da452910fb116888f491d494ddc1772a0d30",
      "git_blob_oid": "014e71b848891d04f00c7ea89d55ff313e2ae83d",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/content_builder.py"
    },
    "src/build_prompt/content_builder/prompts.py": {
      "bytes": 11286,
      "sha256": "3b115cadeb3abcbb2efeafa70863efa9b97e5133773f2dcf7ce29be2108c9a18",
      "git_blob_oid": "ef06b7466204e80f2b1fa37ef535f4a6c72110c3",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/prompts.py"
    },
    "src/build_prompt/content_builder/token_counter.py": {
      "bytes": 10216,
      "sha256": "c47e3a552d230c2941cb135fc03d473d5417c42f5060e31b55be59b50e7289e8",
      "git_blob_oid": "5cde310745a8f1d9aaed0cd807c838a9a617f862",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/token_counter.py"
    },
    "src/build_prompt/content_builder/utils.py": {
      "bytes": 5074,
      "sha256": "888c0a34a6702510f0df98a024c8c546fbfbfcc9f643800f008c730d3110bbd6",
      "git_blob_oid": "5d7290ec90e3f04699de6aa09320520c2e75d351",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/utils.py"
    },
    "src/build_prompt/excel_extract/excel_shot_generator.py": {
      "bytes": 13630,
      "sha256": "927fab986b7860d63ea29f1ef152b4defb5feb96c5020c8a5aebf212547aa82a",
      "git_blob_oid": "57801e60be455abbe7b6d43941ed14872e6ce65a",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/excel_extract/excel_shot_generator.py"
    },
    "src/call_gpt_judge.py": {
      "bytes": 23754,
      "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a",
      "git_blob_oid": "bfff00e9dc35160748e13dcda1e2b2db145b7ebe",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py"
    },
    "src/organize_files.py": {
      "bytes": 14322,
      "sha256": "ce86f2869d8fb01cbca145a287346d4b53e8d55903cec84dd2121344d2c80928",
      "git_blob_oid": "a791406c23b01e9df5619be29dd0bd801782b37c",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/organize_files.py"
    },
    "src/preprocessor/preprocessor_excel.py": {
      "bytes": 22018,
      "sha256": "053da218ce6cc4823f72a454948d02ae47a6b9cd70e05b554054113a8b2cc2ad",
      "git_blob_oid": "d8a22553bf65665bb0aff967c37bed8e42376d97",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_excel.py"
    },
    "src/preprocessor/preprocessor_main.py": {
      "bytes": 16253,
      "sha256": "fe564cfb99d9c53dc6288f1554b4479bfc23194d470f6fd4223b59491d466582",
      "git_blob_oid": "0ebeeac440523d3e6d531b80988b79c445ecdc1f",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_main.py"
    },
    "src/prompt_build_pipeline.py": {
      "bytes": 3939,
      "sha256": "5e470fe0ca683ac008aeb77716b61668c3c688c743863b7993d070fdcf03ae27",
      "git_blob_oid": "f02f216858c9e6d2c96d9aecc2afc606f18173ad",
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/prompt_build_pipeline.py"
    }
  },
  "entrypoints": [
    {
      "role": "organize",
      "path": "src/organize_files.py",
      "symbol": "FileOrganizer.process_entry",
      "line": 187,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/organize_files.py#L187",
      "sha256": "ce86f2869d8fb01cbca145a287346d4b53e8d55903cec84dd2121344d2c80928"
    },
    {
      "role": "pipeline",
      "path": "src/prompt_build_pipeline.py",
      "symbol": "main",
      "line": 23,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/prompt_build_pipeline.py#L23",
      "sha256": "5e470fe0ca683ac008aeb77716b61668c3c688c743863b7993d070fdcf03ae27"
    },
    {
      "role": "preprocess",
      "path": "src/preprocessor/preprocessor_main.py",
      "symbol": "PreprocessorManager.process_task_directory",
      "line": 195,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_main.py#L195",
      "sha256": "fe564cfb99d9c53dc6288f1554b4479bfc23194d470f6fd4223b59491d466582"
    },
    {
      "role": "prompt_builder",
      "path": "src/build_prompt/content_builder/content_builder.py",
      "symbol": "ContentBuilder.build_task",
      "line": 161,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/content_builder.py#L161",
      "sha256": "69122ea7318818883c0628fbe583da452910fb116888f491d494ddc1772a0d30"
    },
    {
      "role": "judge_prompts",
      "path": "src/build_prompt/content_builder/prompts.py",
      "symbol": "JUDGE_PROMPT_EXCEL / JUDGE_PROMPT_GENERAL",
      "line": 6,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/prompts.py#L6",
      "sha256": "3b115cadeb3abcbb2efeafa70863efa9b97e5133773f2dcf7ce29be2108c9a18"
    },
    {
      "role": "model_judge",
      "path": "src/call_gpt_judge.py",
      "symbol": "GPTJudgeCaller.call_api",
      "line": 172,
      "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py#L172",
      "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a"
    }
  ],
  "review_limits": [
    {
      "id": "FINCH-E01",
      "scope": "workflow",
      "workflow_ids": [
        "3",
        "108"
      ],
      "flag": "first_source_excel_only",
      "note": "The content builder selects only the first Excel file for each source/reference/output side. IDs 3 and 108 each have two source XLSX inputs; ID 108 explicitly requests use of the second. Excel preprocessing recalculates files but returns no textual evidence entries.",
      "interpretation": "Record source-visibility limits; verify all required inputs appear in any future judge payload. This does not establish task infeasibility or a measured grading error.",
      "refs": [
        {
          "path": "src/build_prompt/content_builder/content_builder.py",
          "line": 171,
          "end_line": 261,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/content_builder.py#L171-L261",
          "sha256": "69122ea7318818883c0628fbe583da452910fb116888f491d494ddc1772a0d30"
        },
        {
          "path": "src/build_prompt/content_builder/utils.py",
          "line": 64,
          "end_line": 75,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/utils.py#L64-L75",
          "sha256": "888c0a34a6702510f0df98a024c8c546fbfbfcc9f643800f008c730d3110bbd6"
        },
        {
          "path": "src/preprocessor/preprocessor_excel.py",
          "line": 305,
          "end_line": 332,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_excel.py#L305-L332",
          "sha256": "053da218ce6cc4823f72a454948d02ae47a6b9cd70e05b554054113a8b2cc2ad"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E02",
      "scope": "workflow",
      "workflow_ids": [
        "4",
        "5",
        "7",
        "16",
        "17",
        "18",
        "51",
        "52",
        "72",
        "73",
        "81",
        "156",
        "160",
        "161",
        "163",
        "164"
      ],
      "flag": "default_source_non_excel_not_in_judge",
      "note": "Default preprocessing skips source PDF/Markdown/DOCX/images. Sixteen records have such inputs. The top-level pipeline does not forward the opt-in source-non-Excel flag.",
      "interpretation": "State the chosen preprocessing policy and retain missing-evidence diagnostics; reference-based judging may still occur, but should not be described as inspection of every input.",
      "refs": [
        {
          "path": "src/preprocessor/preprocessor_main.py",
          "line": 177,
          "end_line": 193,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_main.py#L177-L193",
          "sha256": "fe564cfb99d9c53dc6288f1554b4479bfc23194d470f6fd4223b59491d466582"
        },
        {
          "path": "src/preprocessor/preprocessor_main.py",
          "line": 420,
          "end_line": 441,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_main.py#L420-L441",
          "sha256": "fe564cfb99d9c53dc6288f1554b4479bfc23194d470f6fd4223b59491d466582"
        },
        {
          "path": "src/prompt_build_pipeline.py",
          "line": 70,
          "end_line": 80,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/prompt_build_pipeline.py#L70-L80",
          "sha256": "5e470fe0ca683ac008aeb77716b61668c3c688c743863b7993d070fdcf03ae27"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E03",
      "scope": "workflow",
      "workflow_ids": [
        "17",
        "18",
        "73"
      ],
      "flag": "source_csv_json_not_in_judge",
      "note": "The preprocessing chain handles PDF, Markdown, Word, Excel and images, with no CSV/JSON handler. The content builder attaches source content through the first Excel path; its general-side builder covers reference/output, not source files. IDs 17, 18 and 73 contain CSV/JSON inputs.",
      "interpretation": "Do not claim the current default judge directly inspects those structured inputs. Keep reference text and original source links; assess coverage before any benchmark run.",
      "refs": [
        {
          "path": "src/preprocessor/preprocessor_main.py",
          "line": 113,
          "end_line": 126,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_main.py#L113-L126",
          "sha256": "fe564cfb99d9c53dc6288f1554b4479bfc23194d470f6fd4223b59491d466582"
        },
        {
          "path": "src/build_prompt/content_builder/content_builder.py",
          "line": 205,
          "end_line": 261,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/content_builder.py#L205-L261",
          "sha256": "69122ea7318818883c0628fbe583da452910fb116888f491d494ddc1772a0d30"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E04",
      "scope": "global",
      "workflow_ids": [],
      "flag": "finch-e04",
      "note": "The organizer skips a workflow when no matching model output file exists; downstream builder processes only created task directories.",
      "interpretation": "Use the full 172-ID denominator and explicitly distinguish missing outputs from judge errors and actual score-zero outcomes.",
      "refs": [
        {
          "path": "src/organize_files.py",
          "line": 198,
          "end_line": 209,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/organize_files.py#L198-L209",
          "sha256": "ce86f2869d8fb01cbca145a287346d4b53e8d55903cec84dd2121344d2c80928"
        },
        {
          "path": "src/build_prompt/content_builder/content_builder.py",
          "line": 112,
          "end_line": 156,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/content_builder.py#L112-L156",
          "sha256": "69122ea7318818883c0628fbe583da452910fb116888f491d494ddc1772a0d30"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E05",
      "scope": "global",
      "workflow_ids": [],
      "flag": "finch-e05",
      "note": "API failures after retries yield score=null; malformed/missing required judge fields yield score=0. Resume detection uses only task_id from an existing results workbook and includes errored rows. Parsed scores are accepted without enforcing a binary numeric domain.",
      "interpretation": "Keep failure reasons and raw judge provenance; validate scores and pin input/output/prompt/model/settings before resuming or aggregating. No measured pass rate is inferred.",
      "refs": [
        {
          "path": "src/call_gpt_judge.py",
          "line": 124,
          "end_line": 129,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py#L124-L129",
          "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a"
        },
        {
          "path": "src/call_gpt_judge.py",
          "line": 172,
          "end_line": 205,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py#L172-L205",
          "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a"
        },
        {
          "path": "src/call_gpt_judge.py",
          "line": 267,
          "end_line": 325,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py#L267-L325",
          "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a"
        },
        {
          "path": "src/call_gpt_judge.py",
          "line": 384,
          "end_line": 419,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/call_gpt_judge.py#L384-L419",
          "sha256": "8cf11aa807f8f3ca33a70d0f7177bfc16feddad075b1f4620819a7e5deb1b28a"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E06",
      "scope": "global",
      "workflow_ids": [],
      "flag": "finch-e06",
      "note": "Judge input is reduced by 40-image / 200000-text-character limits; truncation removes content from the end. Spreadsheet screenshots also use row/column limits and skip hidden sheets by default.",
      "interpretation": "Report the exact evidence retained and truncation/visibility state; this is a model-judge workflow, not exhaustive deterministic validation.",
      "refs": [
        {
          "path": "src/build_prompt/content_builder/config.py",
          "line": 11,
          "end_line": 15,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/config.py#L11-L15",
          "sha256": "e70b601507a4ef188094f61d62204b2b63652b44ef4df54562b8cc99f8f54e74"
        },
        {
          "path": "src/build_prompt/content_builder/token_counter.py",
          "line": 135,
          "end_line": 285,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/content_builder/token_counter.py#L135-L285",
          "sha256": "c47e3a552d230c2941cb135fc03d473d5417c42f5060e31b55be59b50e7289e8"
        },
        {
          "path": "src/build_prompt/excel_extract/excel_shot_generator.py",
          "line": 65,
          "end_line": 88,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/build_prompt/excel_extract/excel_shot_generator.py#L65-L88",
          "sha256": "927fab986b7860d63ea29f1ef152b4defb5feb96c5020c8a5aebf212547aa82a"
        }
      ],
      "runtime_observed": false
    },
    {
      "id": "FINCH-E07",
      "scope": "global",
      "workflow_ids": [],
      "flag": "finch-e07",
      "note": "Excel preprocessing may repair/rewrite/recalculate and save evaluation copies. Main is a newer unified evaluator; README points to the previous branch for original-paper code.",
      "interpretation": "Retain pre/post-processing artifact hashes for any later run; do not label current-main static review a reproduction of paper results.",
      "refs": [
        {
          "path": "src/preprocessor/preprocessor_excel.py",
          "line": 149,
          "end_line": 175,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_excel.py#L149-L175",
          "sha256": "053da218ce6cc4823f72a454948d02ae47a6b9cd70e05b554054113a8b2cc2ad"
        },
        {
          "path": "src/preprocessor/preprocessor_excel.py",
          "line": 211,
          "end_line": 230,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_excel.py#L211-L230",
          "sha256": "053da218ce6cc4823f72a454948d02ae47a6b9cd70e05b554054113a8b2cc2ad"
        },
        {
          "path": "src/preprocessor/preprocessor_excel.py",
          "line": 305,
          "end_line": 332,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/src/preprocessor/preprocessor_excel.py#L305-L332",
          "sha256": "053da218ce6cc4823f72a454948d02ae47a6b9cd70e05b554054113a8b2cc2ad"
        },
        {
          "path": "README.md",
          "line": 1,
          "end_line": 1,
          "source_url": "https://github.com/FinWorkBench/Finch/blob/95a8b8d135a528b325be003e54c55f886a22602d/README.md",
          "sha256": "c0988d6c61c909633859efcf95463550bfeb62319d45fd7b8ab1742746e610ca"
        }
      ],
      "runtime_observed": false
    }
  ]
}