{
  "schema_version": 1,
  "source_id": "scuba",
  "repository": "https://github.com/SalesforceAIResearch/SCUBA",
  "revision": "b893e2239b035f93d5753a32557346d15935d596",
  "reviewed_on": "2026-10-17",
  "status": "STATIC_SOURCE_REVIEW_NOT_RUN",
  "scope_en": "Revision-bound checks of assignment-rule evaluator coverage. Four records need review; the remaining 296 preserve source evidence without implying grader validation.",
  "scope_zh": "固定版本分派规则评分覆盖静态检查。4 条记录待审查；其余 296 条保留源码证据，不表示评分器已验证。",
  "hash_encoding": {
    "query": "UTF-8 bytes of the exact canonical query string",
    "ground_truth": "UTF-8 JSON, sort_keys=true, ensure_ascii=false, separators=(comma,colon)"
  },
  "evidence_files": {
    "data/test_zero_shot.json": {
      "bytes": 637174,
      "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca",
      "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json"
    },
    "scuba/phases/evaluation/milestone_evaluator_service.py": {
      "bytes": 30152,
      "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e",
      "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py"
    },
    "scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json": {
      "bytes": 6075,
      "sha256": "8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351",
      "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json"
    },
    "scuba/phases/evaluation/master_evaluator.py": {
      "bytes": 5276,
      "sha256": "9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4",
      "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py"
    }
  },
  "findings": {
    "condition_precedence_not_checked": {
      "reason_en": "The query requires the description condition to be evaluated first. This evaluator checks condition membership and Boolean logic, but not condition precedence or rule-entry order.",
      "reason_zh": "指令要求优先判断 description 条件；评分器检查条件是否存在及布尔逻辑，不检查条件优先级或规则条目顺序。",
      "evidence": [
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 109,
          "line_end": 127,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        },
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 136,
          "line_end": 169,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        }
      ]
    },
    "numeric_sort_order_not_checked": {
      "reason_en": "The query explicitly requires sort order 1. This evaluator never reads the requested sort_order or compares returned rule-entry order.",
      "reason_zh": "指令明确要求 sort order 为 1；评分器未读取要求中的 sort_order，也未比较返回的规则条目顺序。",
      "evidence": [
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 109,
          "line_end": 127,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        },
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 136,
          "line_end": 169,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        }
      ]
    },
    "requested_accounts_checked_case_rules": {
      "reason_en": "The query says to assign accounts, while retrieval fixes AssignmentRules to Case. Account.Rating is checked as a rule criterion; Account ownership is not checked. A related-account condition on a Case rule would not establish that the requested accounts were assigned.",
      "reason_zh": "指令要求分派 accounts，但检索固定为 Case 的 AssignmentRules。Account.Rating 仅作为规则条件核对，不检查 Account 所有者；Case 规则中的关联账户条件不等于完成账户分派。",
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10640,
          "line_end": 10678,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10640-L10678",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        },
        {
          "source_path": "scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json",
          "line_start": 39,
          "line_end": 48,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json#L39-L48",
          "sha256": "8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351"
        },
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 83,
          "line_end": 115,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83-L115",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        }
      ]
    },
    "empty_connector_cannot_pass": {
      "reason_en": "This record has an empty logic_operator. Only AND and OR reach a successful connector branch; the 0.2-weight connector milestone is therefore false on the normal return path.",
      "reason_zh": "此记录的 logic_operator 为空；只有 AND、OR 能进入连接符成功分支，因此正常返回路径中权重为 0.2 的连接符里程碑必为 false。",
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10644,
          "line_end": 10656,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10644-L10656",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        },
        {
          "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
          "line_start": 144,
          "line_end": 169,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L144-L169",
          "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
        },
        {
          "source_path": "scuba/phases/evaluation/master_evaluator.py",
          "line_start": 106,
          "line_end": 117,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L106-L117",
          "sha256": "9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"
        }
      ]
    }
  },
  "records": [
    {
      "original_id": "service_002_001",
      "source_json_pointer": "/250",
      "status": "needs_review",
      "query_sha256": "60a0d7c7fcaedbc49bf4506eb286f411e45afd66853fa92c7d2147337b4cea11",
      "ground_truth_sha256": "cffce925de876f2e434f3029412508a403b77f73f34cbda03254c9d3d4d9b478",
      "flags": [
        "condition_precedence_not_checked"
      ],
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10460,
          "line_end": 10481,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10460-L10481",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        }
      ]
    },
    {
      "original_id": "service_002_003",
      "source_json_pointer": "/252",
      "status": "needs_review",
      "query_sha256": "401fc7bb04b5da7883d0cd3e0213cec62ed9d43ca329da979607e48a355ba010",
      "ground_truth_sha256": "8d87b96762b0ed3dacb0f901c9ec8489fa00cac7c4fae8310f802058d95676f8",
      "flags": [
        "condition_precedence_not_checked"
      ],
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10580,
          "line_end": 10601,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10580-L10601",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        }
      ]
    },
    {
      "original_id": "service_002_004",
      "source_json_pointer": "/253",
      "status": "needs_review",
      "query_sha256": "a820723ba77b17be9bd7bf3ad040951feec4cd327533dac6ea82e5b948cd69d9",
      "ground_truth_sha256": "92c9b2e39ace2e26383f27a5b09002900bb79194b6354cb51ff70549011c6138",
      "flags": [
        "requested_accounts_checked_case_rules",
        "empty_connector_cannot_pass",
        "numeric_sort_order_not_checked"
      ],
      "static_score_upper_bound": {
        "value": 0.8,
        "kind": "normal_return_weight_sum_bound",
        "runtime_score": false,
        "scope_en": "Static bound for the pinned normal-return path only. ScoreCard requires 1 for completion. This is not a measured run score or a claim that the business request is impossible; exceptions have a separate -1 path.",
        "scope_zh": "仅针对固定版本正常返回路径的静态上限；ScoreCard 需达到 1 才判完成。这不是实测分数，也不代表业务请求不可行；异常使用另一个 -1 路径。",
        "evidence": [
          {
            "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
            "line_start": 97,
            "line_end": 103,
            "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L97-L103",
            "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
          },
          {
            "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
            "line_start": 130,
            "line_end": 141,
            "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L130-L141",
            "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
          },
          {
            "source_path": "scuba/phases/evaluation/milestone_evaluator_service.py",
            "line_start": 144,
            "line_end": 169,
            "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L144-L169",
            "sha256": "dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"
          },
          {
            "source_path": "scuba/phases/evaluation/master_evaluator.py",
            "line_start": 13,
            "line_end": 18,
            "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L13-L18",
            "sha256": "9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"
          },
          {
            "source_path": "scuba/phases/evaluation/master_evaluator.py",
            "line_start": 106,
            "line_end": 117,
            "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L106-L117",
            "sha256": "9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"
          }
        ]
      },
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10640,
          "line_end": 10656,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10640-L10656",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        }
      ]
    },
    {
      "original_id": "service_002_005",
      "source_json_pointer": "/254",
      "status": "needs_review",
      "query_sha256": "5fb2d70d5cc7fe5a41d371039c3bec4813463efac67535c90627c8e89d970296",
      "ground_truth_sha256": "e9e9878da2d781e78991b553c87eceba2e4b8f54c981ef1799afe6d84c3586bf",
      "flags": [
        "numeric_sort_order_not_checked"
      ],
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10695,
          "line_end": 10716,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10695-L10716",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        }
      ]
    }
  ],
  "unflagged_observations": [
    {
      "original_id": "service_002_002",
      "status": "source_preserved",
      "query_sha256": "1525a8dc71ab175dc41f3f3c0a82cf8c498b83124469878714f25728ec837dad",
      "ground_truth_sha256": "bd0061e94919c9369e251d82debe6f2d8ddf966376a95e9e7edb7ba02f1b58d3",
      "reason_en": "Ground truth contains sort_order=1, but the canonical query does not require ordering. This annotation alone is not treated as a defect in its requested goal.",
      "reason_zh": "ground truth 含 sort_order=1，但规范指令没有顺序要求；不凭该注释把任务目标标为有缺陷。",
      "evidence": [
        {
          "source_path": "data/test_zero_shot.json",
          "line_start": 10520,
          "line_end": 10542,
          "source_url": "https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10520-L10542",
          "sha256": "23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"
        }
      ]
    }
  ],
  "limits_en": [
    "No upstream code, org setup or grader was executed.",
    "The Accounts/Case ambiguity is separate from the deterministic blank-connector control-flow observation; platform feasibility was not tested.",
    "Condition precedence and numeric rule-entry sort order are different constraints.",
    "Original canonical and same-ID demonstration objects are unchanged; the 36-file source snapshot is unchanged."
  ],
  "limits_zh": [
    "未执行上游代码、部署 org 或运行评分器。",
    "Accounts／Case 歧义与空连接符控制流问题分别记录；未测试平台可行性。",
    "条件优先级与规则条目的数值 sort order 是不同约束。",
    "规范任务与同 ID 演示对象、包含 36 份文件的来源快照均保持原样。"
  ]
}