{ "schema_version": 1, "source_id": "windowsworld", "reviewed_at": "2026-10-17T06:42:19.654514+00:00", "status": "imported_source_preserved_not_run", "collection_status": "imported", "executed_tasks": 0, "repository": "https://github.com/HITsz-TMG/WindowsWorld", "revision": "fbccd464f94fec9e284e139f97bf96d0b192f580", "paper": "https://arxiv.org/abs/2604.27776v1", "scope": "Independent static recheck of all 181 task records and selected pinned runner, CoAct, environment and result-handling paths. License header/copyright, README license/provenance declarations and installation dependencies inspected; files hashed in full. Paper v1 task-level definitions and Figure 2 rechecked. No upstream module imported, source executed, environment installed, model called or VM downloaded.", "license": { "declared_spdx": "Apache-2.0", "declaration_refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/README.md#L4", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/LICENSE" ], "preserved_copyright_notice": "Copyright 2024 XLANG NLP Lab", "upstream_attribution_ref": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/README.md#L162-L168", "snapshot_requirement": "Preserve the complete original LICENSE and README acknowledgement; source-code license is not a grant to redistribute Windows or third-party application binaries." }, "counts": { "definitions": 181, "unique_task_ids": 181, "duplicate_task_id_groups": 0, "levels": { "L1": 39, "L2": 80, "L3": 50, "L4": 12 }, "task_completion_definitions": 169, "author_labeled_infeasible_definitions": 12, "feasibility_runtime_verified": 0, "raw_persona_labels": 16, "raw_application_labels": 22, "multi_app_records": 141, "per_record_raw_app_list_length": { "1": 40, "2": 43, "3": 86, "4": 10, "5": 2 }, "per_record_distinct_app_label_count": { "1": 40, "2": 44, "3": 85, "4": 10, "5": 2 }, "records_with_repeated_app_labels": 1, "source_intermediate_check_strings": 899, "source_intermediate_check_strings_by_level": { "L1": 144, "L2": 436, "L3": 314, "L4": 5 }, "l1_l3_eligible_not_executed_intermediate_check_strings": 894, "l4_bypassed_intermediate_check_strings": 5, "source_assertion_strings_not_executed_by_evaluate": 430, "upstream_review_status": { "approved": 126, "pending": 55 }, "l4_upstream_review_status": { "pending": 12 }, "upstream_validity_info_is_valid": { "true": 160, "false": 21 }, "instructions_present": { "instruction": 181, "instruction_cn": 181 }, "exact_unique_instruction_strings": 181, "exact_unique_instruction_cn_strings": 181, "active_instruction_languages": { "en": 179, "zh-CN": 2 } }, "application_counts": { "raw_label_occurrences": { "Adobe Acrobat": 10, "Adobe Reader (or default PDF viewer)": 1, "Calculator": 7, "Chrome": 70, "Control Panel": 4, "Excel": 73, "File Explorer": 59, "GIMP": 14, "JSONFormatter": 1, "Outlook": 72, "Paint": 11, "Photos": 7, "PowerPoint": 18, "PowerShell": 2, "Snipping Tool": 1, "Task Manager": 2, "VLC Player": 1, "VS Code": 30, "WinZip/Windows内置解压工具": 1, "Windows Terminal": 9, "Word": 40, "cmd": 1 }, "task_records_per_distinct_raw_label": { "Adobe Acrobat": 10, "Adobe Reader (or default PDF viewer)": 1, "Calculator": 7, "Chrome": 70, "Control Panel": 4, "Excel": 73, "File Explorer": 59, "GIMP": 14, "JSONFormatter": 1, "Outlook": 72, "Paint": 11, "Photos": 7, "PowerPoint": 18, "PowerShell": 2, "Snipping Tool": 1, "Task Manager": 2, "VLC Player": 1, "VS Code": 29, "WinZip/Windows内置解压工具": 1, "Windows Terminal": 9, "Word": 40, "cmd": 1 }, "grain_note": "Raw occurrences retain repeated entries; record counts use set(involved_apps) within each task. Neither is a normalized product inventory.", "paper_reported_apps": 17, "canonical_products": null }, "effective_evaluation": { "L1-L3": { "intent": "task_completion", "kind": "model_judge_instruction_actions_screenshots", "model_declared": "qwen3-vl-plus", "configured_criteria_state": "eligible_not_executed", "read_fields": [ "instruction", "evaluation_metrics.intermediate_checks", "evaluation_metrics.success_criterion" ], "ignored_by_entrypoint_fields": [ "evaluation_metrics.assertions", "evaluation_metrics.expected_final_state" ], "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L458-L521" ] }, "L4": { "intent": "infeasibility_recognition", "kind": "terminal_failure_signal_heuristic", "accepted_signal_conditions": [ "actions is empty or None", "last action is a string containing fail case-insensitively", "last action is a dictionary whose action_type lowercases to fail" ], "criterion_state": "configured_criteria_bypassed", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L424-L456" ] }, "standard_error_boundary": "Per-action errors are logged without synthesizing FAIL. Reset/prediction exceptions are outside that catch and may propagate to the worker. No runtime result is established by this review.", "error_refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L308-L396", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/main.py#L157-L188" ] }, "instruction_contract": { "active_agent_field": "instruction", "active_model_judge_field": "instruction", "result_header_field": "instruction_cn", "preserve_both": true, "translation_equivalence": "not_assumed", "confirmed_semantic_difference": { "task_id": "win_fin_l4_002", "json_pointer": "/148", "instruction_provider": "HeyWhale", "instruction_cn_provider": "Kaggle", "instruction": "Download the open-source stock price dataset from HeyWhale, import it into Excel to calculate the 30-day moving average using formulas, then copy the resulting chart into a new PowerPoint presentation titled 'Stock_Analysis_Q2.pptx'.", "instruction_cn": "从 https://www.kaggle.com/datasets/parthsojitra/google-stock-prize-of-10-years 下载开源股票价格数据集，导入Excel并使用公式计算30日移动平均线，然后将生成的图表复制到新建的PowerPoint演示文稿中，文件命名为“Stock_Analysis_Q2.pptx”。" }, "audit_limit": "One provider mismatch explicitly reviewed; no claim that all other pairs are equivalent.", "active_field_language_review": { "reviewed_at": "2026-10-17T08:12:58.541683+00:00", "revision": "fbccd464f94fec9e284e139f97bf96d0b192f580", "counts": { "en": 179, "zh-CN": 2 }, "chinese_record_ids": [ "win_fin_l2_008", "win_it__l1_001" ], "chinese_instruction_sha256": { "win_fin_l2_008": "f2ae630fc4bee5a93176073ef0aa25ab44a31e27113cf85a2cecbdb5ec8408fa", "win_it__l1_001": "4516e84bca1b04ffe46bf4078a083c7a58df11fc01890320e815ace286e9ec4b" }, "english_instructions_with_chinese_filenames_or_terms": 21, "method": "Read the pinned active instruction fields. Two complete Chinese instructions are annotated zh-CN; Chinese filenames or terms inside English instructions do not change their language. Original fields are not translated or replaced.", "scope": "Language annotation only; translation equivalence and task execution are not established." } }, "environment": { "os_in_runner": "Windows", "provider": "vmware", "screen_size": [ 1920, 1080 ], "declared_action_spaces": [ "pyautogui", "computer_13" ], "prepared_snapshot": "init_state", "external_vm_files": [ "Windows0.mf", "Windows0.ovf", "Windows0-disk1.vmdk", "Windows0-file1.nvram" ], "vm_downloaded": false, "vm_launched": false, "third_party_applications_verified": false, "live_external_urls_verified": false, "linked_vm_dataset": { "repository": "https://huggingface.co/datasets/HalfCooler/WindowsWorld", "revision": "b271ded96b25b0c67918d52cb2895572fdb27004", "metadata_and_file_names_rechecked": true, "declared_license": "apache-2.0", "files_present": [ ".gitattributes", "README.md", "Windows0-disk1.vmdk", "Windows0-file1.nvram", "Windows0.mf", "Windows0.ovf" ], "scope": "VM archive/support files, not another 181 task-definition set. No binary download or redistribution." }, "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L554-L562", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/Installation%20Guide.md#L65-L82" ] }, "issues": [ { "id": "WW-L4-SIGNAL-ONLY", "scope": "all L4 records", "affected_task_ids": [ "win_soc_l4_001", "win_des_l4_001", "win_fin_l4_001", "win_fin_l4_002", "win_fin_l4_003", "win_fin_l4_004", "win_con_l4_001", "win_it_l4_004", "win_it__l4_001", "win_gen_l4_003", "win_cus_l4_001", "win_cus_l4_002" ], "finding_en": "L4 is an author-labeled infeasibility subset. The scoring branch accepts empty actions, a final string containing fail case-insensitively, or a final dictionary action_type equal to fail. It does not inspect screenshots, error reasons, environment feasibility or configured task criteria.", "finding_zh": "L4 为上游标记的不可行任务；评分接受空动作、末尾字符串包含 fail 或末尾 action_type 为 fail，不核验截图、失败原因、环境可行性或逐题标准。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L424-L456", "https://arxiv.org/html/2604.27776v1#S3.SS2" ], "import_action": "Preserve L4 as infeasibility-recognition intent with a terminal-signal heuristic verifier; do not count it as ordinary completion or verified infeasibility." }, { "id": "WW-COACT-EXCEPTION-FAIL", "scope": "CoAct wrapper path for L4 records", "affected_task_ids": [ "win_soc_l4_001", "win_des_l4_001", "win_fin_l4_001", "win_fin_l4_002", "win_fin_l4_003", "win_fin_l4_004", "win_con_l4_001", "win_it_l4_004", "win_it__l4_001", "win_gen_l4_003", "win_cus_l4_001", "win_cus_l4_002" ], "finding_en": "The CoAct wrapper catches exceptions in run/collection, writes error.txt and sets actions=[FAIL], then calls evaluate. This reaches the same L4 acceptance branch as an intentional failure signal.", "finding_zh": "CoAct 外层捕获运行／收集异常后记录 error.txt 并设置动作 FAIL，再调用评分；L4 会将此信号与主动识别不可行一并接受。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L236-L270", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L424-L456" ], "import_action": "Preserve the source path and a review note; separate runtime errors from outcome scores in any later execution. This audit reports no incidence or exploit rate." }, { "id": "WW-COACT-RETURNED-ERROR", "scope": "CoAct adapter and wrapper", "affected_task_ids": [ "win_soc_l4_001", "win_des_l4_001", "win_fin_l4_001", "win_fin_l4_002", "win_fin_l4_003", "win_fin_l4_004", "win_con_l4_001", "win_it_l4_004", "win_it__l4_001", "win_gen_l4_003", "win_cus_l4_001", "win_cus_l4_002" ], "finding_en": "The adapter may return an error object after its retry loop; the wrapper stores result_info but does not branch on its error field before grading environment history. L4 acceptance depends on that history being empty or ending in an accepted failure signal.", "finding_zh": "适配器重试后可返回 error 对象；外层保存对象但未据 error 字段停止评分，仍按环境动作历史判断，L4 仅在空历史或末尾失败信号时接受。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/mm_agents/coact_agent.py#L304-L319", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L238-L270" ], "import_action": "Describe this conditional path precisely; do not claim every CoAct error earns success." }, { "id": "WW-BILINGUAL-PROVIDER-DIFFERENCE", "scope": "manually confirmed bilingual example; not an exhaustive translation audit", "affected_task_ids": [ "win_fin_l4_002" ], "finding_en": "At JSON pointer /148, instruction names HeyWhale while instruction_cn names a kaggle.com dataset URL. Both agent paths and the model-judge prompt use instruction; result text headers use instruction_cn.", "finding_zh": "JSON /148 的英文指令指定 HeyWhale，中文指定 Kaggle 数据集；代理及 judge 使用 instruction，而结果标题使用 instruction_cn。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L238", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L288", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L340", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L415", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L494" ], "import_action": "Keep instruction as the active upstream prompt and preserve instruction_cn independently. Do not silently treat the two as equivalent translations." }, { "id": "WW-DUPLICATED-APP-LABEL", "scope": "per-record app-label grain", "affected_task_ids": [ "win_sof_l3_005" ], "finding_en": "The involved_apps list contains Chrome, VS Code, VS Code. VS Code therefore occurs 30 times in raw lists but in only 29 distinct task records.", "finding_zh": "involved_apps 为 Chrome、VS Code、VS Code；VS Code 原始出现 30 次，但只涉及 29 条任务。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json" ], "import_action": "Preserve the raw list; de-duplicate within each record for application coverage statistics." }, { "id": "WW-PAPER-APP-COUNT-GRAIN", "scope": "source inventory", "affected_task_ids": [], "finding_en": "The README/paper reports 17 apps while the JSON has 22 exact raw labels. Paper Figure 2 lists Thunderbird=72 and PowerShell=18; JSON raw labels are Outlook=72 and PowerShell=2. The discrepancy is not resolved by treating these distinct products as aliases.", "finding_zh": "README／论文报告 17 个应用，JSON 有 22 种原始标签；论文图 2 的 Thunderbird=72、PowerShell=18 与 JSON 的 Outlook=72、PowerShell=2 尚未对齐。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/README.md#L13-L18", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json", "https://arxiv.org/html/2604.27776v1#S2.F2" ], "import_action": "Label 22 as raw labels, not confirmed unique software products; retain paper-reported counts separately." }, { "id": "WW-L4-CONFIGURED-CHECKS-BYPASSED", "scope": "source-configured criteria", "affected_task_ids": [ "win_soc_l4_001" ], "finding_en": "Five intermediate-check strings occur in this L4 definition. The early-return L4 branch bypasses them. Across the complete corpus, 430 assertion strings are not read or executed by hf_run.evaluate.", "finding_zh": "此 L4 定义含 5 条中间标准，被 L4 提前返回分支跳过；全库 430 条 assertions 字符串也不由 hf_run.evaluate 读取或执行。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L424-L521" ], "import_action": "Keep raw criteria, but distinguish 894 L1–L3 eligible judge strings from 5 bypassed L4 strings and non-executable assertion text." }, { "id": "WW-L4-FOLLOWUP-DELIVERABLE-NOT-CHECKED", "scope": "manually confirmed rubric/entrypoint mismatch example", "affected_task_ids": [ "win_it__l4_001" ], "finding_en": "The instruction and success_criterion request an email explaining a missing report, but the L4 entrypoint only checks the terminal failure signal and does not verify that follow-up deliverable.", "finding_zh": "该指令及 success_criterion 要求发送文件缺失说明邮件，但 L4 入口仅检查终止失败信号，不验收这项后续交付。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L424-L456" ], "import_action": "Expose the configured objective and effective verifier separately; do not claim all stated outcomes are checked." }, { "id": "WW-UPSTREAM-REVIEW-STATUS", "scope": "source metadata", "affected_task_ids": [ "win_soc_l1_003", "win_soc_l2_001", "win_soc_l2_002", "win_soc_l2_003", "win_soc_l2_004", "win_soc_l2_005", "win_soc_l3_002", "win_soc_l4_001", "win_des_l1_004", "win_des_l1_002", "win_des_l1_005", "win_des_l2_001", "win_des_l2_002", "win_des_l2_005", "win_des_l2_008", "win_des_l3_001", "win_des_l3_005", "win_des_l4_001", "win_fin_l1_001", "win_fin_l1_002", "win_fin_l1_003", "win_fin_l4_001", "win_fin_l4_002", "win_fin_l2_005", "win_fin_l2_004", "win_fin_l3_001", "win_fin_l4_003", "win_fin_l4_004", "win_con_l1_002", "win_con_l1_004", "win_con_l2_003", "win_con_l2_002", "win_con_l2_004", "win_con_l2_006", "win_con_l3_005", "win_con_l3_004", "win_con_l4_001", "win_gen_l2_002", "win_gen_l2_006", "win_gen_l2_005", "win_gen_l3_002", "win_gen_l3_003", "win_it__l1_004", "win_it__l1_005", "win_it__l2_003", "win_it_l4_004", "win_it__l2_006", "win_it__l3_002", "win_it__l4_001", "win_gen_l2_009", "win_gen_l2_010", "win_gen_l4_003", "win_cus_l2_005", "win_cus_l4_001", "win_cus_l4_002" ], "finding_en": "review_status contains 126 approved and 55 pending records. All 12 L4 records are pending. These upstream labels are not this collection’s execution or quality verdicts.", "finding_zh": "上游审核字段为 126 approved、55 pending；12 条 L4 全为 pending。这些是源元数据，不是本项目运行或质量结论。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json" ], "import_action": "Retain upstream review metadata with its provenance; do not silently certify all definitions." }, { "id": "WW-STANDARD-RESET-SCOPE", "scope": "standard runner environment setup", "affected_task_ids": [], "finding_en": "run_one_task calls env.reset() without task_config. DesktopEnv applies task-specific setup only when task_config is supplied. The guide describes a prepared VM and init_state snapshot; the CoAct adapter separately resets with task_config.", "finding_zh": "普通入口调用无 task_config 的 env.reset()；环境仅在收到配置时执行逐题准备。指南依赖预制 VM 与 init_state；CoAct 适配器另有带配置重置。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L308-L314", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/desktop_env/desktop_env.py#L242-L304", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/mm_agents/coact_agent.py#L245-L250", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/Installation%20Guide.md#L65-L82" ], "import_action": "Preserve declared setup as source configuration, not proof that fixtures were built or reproduced." }, { "id": "WW-RESUME-DIRECTORY-CHECK", "scope": "result resumption and aggregation", "affected_task_ids": [], "finding_en": "hf_run.main skips existing task entries in hf_result before requiring a successful result file; task directories are created before execution. Completion and directory existence are distinct.", "finding_zh": "恢复逻辑按 hf_result 中已有任务目录跳过，目录又在执行前创建；目录存在不等于有效结果或任务完成。", "refs": [ "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L537-L552", "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py#L630-L640" ], "import_action": "Do not infer executed or completed counts from result-directory counts." } ], "task_index": [ { "task_id": "win_hr__l1_001", "json_pointer": "/0", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l1_002", "json_pointer": "/1", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_001", "json_pointer": "/2", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_003", "json_pointer": "/3", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_004", "json_pointer": "/4", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_005", "json_pointer": "/5", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_006", "json_pointer": "/6", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l2_008", "json_pointer": "/7", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l3_001", "json_pointer": "/8", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l3_004", "json_pointer": "/9", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l3_005", "json_pointer": "/10", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_hr__l3_007", "json_pointer": "/11", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l1_001", "json_pointer": "/12", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l1_002", "json_pointer": "/13", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l1_003", "json_pointer": "/14", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l2_001", "json_pointer": "/15", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l2_004", "json_pointer": "/16", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l2_005", "json_pointer": "/17", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l2_006", "json_pointer": "/18", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l2_007", "json_pointer": "/19", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l3_001", "json_pointer": "/20", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l3_002", "json_pointer": "/21", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l3_003", "json_pointer": "/22", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l3_004", "json_pointer": "/23", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sof_l3_005", "json_pointer": "/24", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [ "WW-DUPLICATED-APP-LABEL" ], "instruction_language": "en" }, { "task_id": "win_sof_l3_006", "json_pointer": "/25", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l1_001", "json_pointer": "/26", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l1_002", "json_pointer": "/27", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l2_001", "json_pointer": "/28", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l2_005", "json_pointer": "/29", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l2_007", "json_pointer": "/30", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l2_008", "json_pointer": "/31", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l3_001", "json_pointer": "/32", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l3_005", "json_pointer": "/33", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l3_007", "json_pointer": "/34", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_fin_l2_003", "json_pointer": "/35", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_fin_l2_007", "json_pointer": "/36", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_fin_l3_002", "json_pointer": "/37", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_fin_l3_004", "json_pointer": "/38", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l1_001", "json_pointer": "/39", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l1_002", "json_pointer": "/40", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l2_003", "json_pointer": "/41", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l2_005", "json_pointer": "/42", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l2_006", "json_pointer": "/43", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l2_008", "json_pointer": "/44", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l3_005", "json_pointer": "/45", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_dat_l3_007", "json_pointer": "/46", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l1_002", "json_pointer": "/47", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l1_003", "json_pointer": "/48", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l2_001", "json_pointer": "/49", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l2_004", "json_pointer": "/50", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l2_007", "json_pointer": "/51", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l2_008", "json_pointer": "/52", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_mar_l3_007", "json_pointer": "/53", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l1_003", "json_pointer": "/54", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l2_001", "json_pointer": "/55", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l2_005", "json_pointer": "/56", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l2_008", "json_pointer": "/57", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l3_001", "json_pointer": "/58", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_con_l3_002", "json_pointer": "/59", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l1_001", "json_pointer": "/60", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l1_003", "json_pointer": "/61", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l2_002", "json_pointer": "/62", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l2_003", "json_pointer": "/63", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l2_006", "json_pointer": "/64", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l3_004", "json_pointer": "/65", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_pro_l3_006", "json_pointer": "/66", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l1_001", "json_pointer": "/67", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l1_003", "json_pointer": "/68", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l2_001", "json_pointer": "/69", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l2_002", "json_pointer": "/70", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l2_003", "json_pointer": "/71", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l2_004", "json_pointer": "/72", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l2_006", "json_pointer": "/73", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_cus_l3_005", "json_pointer": "/74", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l1_002", "json_pointer": "/75", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l1_003", "json_pointer": "/76", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l2_002", "json_pointer": "/77", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l2_005", "json_pointer": "/78", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l3_001", "json_pointer": "/79", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l3_004", "json_pointer": "/80", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l3_005", "json_pointer": "/81", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l3_006", "json_pointer": "/82", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l1_001", "json_pointer": "/83", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l1_003", "json_pointer": "/84", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l2_001", "json_pointer": "/85", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l2_003", "json_pointer": "/86", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l2_005", "json_pointer": "/87", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l2_006", "json_pointer": "/88", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l2_007", "json_pointer": "/89", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l3_002", "json_pointer": "/90", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l3_004", "json_pointer": "/91", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l3_006", "json_pointer": "/92", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_acc_l3_007", "json_pointer": "/93", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l1_001", "json_pointer": "/94", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l1_003", "json_pointer": "/95", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l2_003", "json_pointer": "/96", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l2_004", "json_pointer": "/97", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l2_006", "json_pointer": "/98", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l2_007", "json_pointer": "/99", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l3_002", "json_pointer": "/100", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l3_003", "json_pointer": "/101", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_des_l3_004", "json_pointer": "/102", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_adm_l1_003", "json_pointer": "/103", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_adm_l2_001", "json_pointer": "/104", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_adm_l2_008", "json_pointer": "/105", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_adm_l3_004", "json_pointer": "/106", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_adm_l3_005", "json_pointer": "/107", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l1_001", "json_pointer": "/108", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l1_002", "json_pointer": "/109", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l2_007", "json_pointer": "/110", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l2_008", "json_pointer": "/111", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l3_004", "json_pointer": "/112", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l1_002", "json_pointer": "/113", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l1_003", "json_pointer": "/114", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l2_001", "json_pointer": "/115", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l2_003", "json_pointer": "/116", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l2_004", "json_pointer": "/117", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l2_008", "json_pointer": "/118", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l3_004", "json_pointer": "/119", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_gen_l3_005", "json_pointer": "/120", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_sal_l3_003", "json_pointer": "/121", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_fin_l2_008", "json_pointer": "/122", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "zh-CN" }, { "task_id": "win_con_l3_003", "json_pointer": "/123", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_it__l1_001", "json_pointer": "/124", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "zh-CN" }, { "task_id": "win_it__l2_007", "json_pointer": "/125", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "approved", "issue_ids": [], "instruction_language": "en" }, { "task_id": "win_soc_l1_003", "json_pointer": "/126", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l2_001", "json_pointer": "/127", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l2_002", "json_pointer": "/128", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l2_003", "json_pointer": "/129", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l2_004", "json_pointer": "/130", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l2_005", "json_pointer": "/131", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l3_002", "json_pointer": "/132", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_soc_l4_001", "json_pointer": "/133", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-L4-CONFIGURED-CHECKS-BYPASSED", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l1_004", "json_pointer": "/134", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l1_002", "json_pointer": "/135", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l1_005", "json_pointer": "/136", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l2_001", "json_pointer": "/137", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l2_002", "json_pointer": "/138", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l2_005", "json_pointer": "/139", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l2_008", "json_pointer": "/140", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l3_001", "json_pointer": "/141", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l3_005", "json_pointer": "/142", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_des_l4_001", "json_pointer": "/143", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l1_001", "json_pointer": "/144", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l1_002", "json_pointer": "/145", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l1_003", "json_pointer": "/146", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l4_001", "json_pointer": "/147", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l4_002", "json_pointer": "/148", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-BILINGUAL-PROVIDER-DIFFERENCE", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l2_005", "json_pointer": "/149", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l2_004", "json_pointer": "/150", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l3_001", "json_pointer": "/151", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l4_003", "json_pointer": "/152", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_fin_l4_004", "json_pointer": "/153", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l1_002", "json_pointer": "/154", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l1_004", "json_pointer": "/155", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l2_003", "json_pointer": "/156", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l2_002", "json_pointer": "/157", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l2_004", "json_pointer": "/158", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l2_006", "json_pointer": "/159", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l3_005", "json_pointer": "/160", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l3_004", "json_pointer": "/161", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_con_l4_001", "json_pointer": "/162", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l2_002", "json_pointer": "/163", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l2_006", "json_pointer": "/164", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l2_005", "json_pointer": "/165", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l3_002", "json_pointer": "/166", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l3_003", "json_pointer": "/167", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l1_004", "json_pointer": "/168", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l1_005", "json_pointer": "/169", "level": "L1", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l2_003", "json_pointer": "/170", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it_l4_004", "json_pointer": "/171", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l2_006", "json_pointer": "/172", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l3_002", "json_pointer": "/173", "level": "L3", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_it__l4_001", "json_pointer": "/174", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-L4-FOLLOWUP-DELIVERABLE-NOT-CHECKED", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l2_009", "json_pointer": "/175", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l2_010", "json_pointer": "/176", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_gen_l4_003", "json_pointer": "/177", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_cus_l2_005", "json_pointer": "/178", "level": "L2", "evaluation_intent": "task_completion", "source_review_status": "pending", "issue_ids": [ "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_cus_l4_001", "json_pointer": "/179", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" }, { "task_id": "win_cus_l4_002", "json_pointer": "/180", "level": "L4", "evaluation_intent": "infeasibility_recognition", "source_review_status": "pending", "issue_ids": [ "WW-L4-SIGNAL-ONLY", "WW-COACT-EXCEPTION-FAIL", "WW-COACT-RETURNED-ERROR", "WW-UPSTREAM-REVIEW-STATUS" ], "instruction_language": "en" } ], "files": { "LICENSE": { "sha256": "a8d1d88a0f9cce3cdbff73b03a80dc5e5d69e3ba2443946c268815a550c9e0ec", "git_blob_oid": "60d0edcb164e63027fda49865712d5bcde225195", "bytes": 11343, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/LICENSE" }, "README.md": { "sha256": "9857bd8ae9f521f152761ae3bd85debe4e7471aad34c9eb47618332948bca6c9", "git_blob_oid": "8d6d4954884d49382a3664d60ac2944688127c1a", "bytes": 7050, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/README.md" }, "benchmark.json": { "sha256": "c894bbce218db9fad5f8c54b9fcdb0e77f15622efc8d14890c20aafbb4a4c551", "git_blob_oid": "adae53ec32fe9b92183fe76f400a4aa4c457e541", "bytes": 531900, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/benchmark.json" }, "hf_run.py": { "sha256": "dc3c4925f4a8fcf065a44438d8763867686db3bbc875500bf3ad869d9b896046", "git_blob_oid": "cd9f92c8208000b1a517a59e216ac9fecb4f3f64", "bytes": 36801, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/hf_run.py" }, "main.py": { "sha256": "2e025934a656bd4bf167abe104af9be4ee18ca6a2968f773c25fbb2889a11fd1", "git_blob_oid": "9a877a0fd5f96c5685cbf14c09542f8b43edc3f2", "bytes": 21890, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/main.py" }, "show_result.py": { "sha256": "dc43f157648c7d58aefbce8479680852f55ae5cac4ffba3fbbc8b999263604be", "git_blob_oid": "db1cd57cc3d24b7949573e1b9ba1b3196b95aaa3", "bytes": 21976, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/show_result.py" }, "desktop_env/desktop_env.py": { "sha256": "fd5716ffe516fb9c01a749c7f6564a86708a403b68999820eba59f9b537a4a55", "git_blob_oid": "aa754211fffc9c6944ffd71caad241e5dcc73e60", "bytes": 25367, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/desktop_env/desktop_env.py" }, "mm_agents/coact_agent.py": { "sha256": "f1eb0d94e37f737017d5400fc681126bfdb17de2deee64aa438f0606da4c2850", "git_blob_oid": "c5b9b34e2234a03cb426045bf9a0b9f88ecbb250", "bytes": 18911, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/mm_agents/coact_agent.py" }, "Installation Guide.md": { "sha256": "936fa60f4dca37e984ba1e9a357c5c5abd7e2ac554748ce4ca8ab087e5258da2", "git_blob_oid": "239a58038a1ab0ef819440e80a2e0e17500a78d7", "bytes": 2827, "url": "https://github.com/HITsz-TMG/WindowsWorld/blob/fbccd464f94fec9e284e139f97bf96d0b192f580/Installation%20Guide.md" } }, "reproducibility": { "input": "benchmark.json at the repository revision above", "input_sha256": "c894bbce218db9fad5f8c54b9fcdb0e77f15622efc8d14890c20aafbb4a4c551", "encoding": "UTF-8 with optional BOM; sha256 covers original bytes", "method": "Standard-library JSON counting and Python AST inspection only; benchmark functions were not called.", "count_recipe_python": "import json\nfrom collections import Counter\nfrom pathlib import Path\nrows = json.loads(Path('benchmark.json').read_text(encoding='utf-8-sig'))\nprint('records', len(rows), 'unique_ids', len({r['task_id'] for r in rows}))\nprint('levels', dict(Counter(r['task_category'] for r in rows)))\nprint('raw_labels', len({a for r in rows for a in r['involved_apps']}))\nprint('raw_label_occurrences', dict(Counter(a for r in rows for a in r['involved_apps'])))\nprint('record_label_counts', dict(Counter(a for r in rows for a in set(r['involved_apps']))))\nprint('per_record_distinct_app_count', dict(Counter(len(set(r['involved_apps'])) for r in rows)))\nprint('multi_app_records', sum(len(set(r['involved_apps'])) > 1 for r in rows))\nprint('checks_by_level', {level: sum(len(r['evaluation_metrics'].get('intermediate_checks', [])) for r in rows if r['task_category'] == level) for level in ['L1', 'L2', 'L3', 'L4']})\nprint('assertion_strings', sum(len(r['evaluation_metrics'].get('assertions', [])) for r in rows))\nprint('source_review_status', dict(Counter(r.get('review_status') for r in rows)))\n", "ast_checks": { "function": "hf_run.evaluate", "lines": [ 420, 521 ], "first_branch_test": "task['task_category'] == 'L4'", "literal_final_result_branches": [ true, false ], "subscript_fields_in_function": [ "evaluation_metrics", "instruction", "intermediate_checks", "success_criterion", "task_category" ] } }, "import_assessment": { "decision": "Imported after collector checks: 181 source objects preserved, 169 completion targets and 12 upstream-marked infeasibility targets separated; previous records and archives unchanged. No task or judge execution.", "preserve": [ "181 original IDs and complete source objects", "instruction as active prompt, plus independent instruction_cn", "upstream task levels, persona strings, review and validity metadata", "source criteria and actual scoring-entrypoint references", "original app labels including the repeated entry, with separate deduplicated coverage statistics", "full LICENSE, README attribution and pinned small source files" ], "exclude_from_snapshot": [ "Windows virtual-machine binaries", "third-party application installers", "live account material", "local execution artifacts" ], "do_not_claim": [ "169 tasks proven feasible or completed", "12 L4 scenarios experimentally verified infeasible", "899 executed checks or 430 executable assertion validators", "22 distinct verified software products", "All bilingual pairs equivalent", "All 181 upstream records approved", "Published model performance reproduced", "A complete independently runnable environment included" ] }, "import_verification": { "checked_at": "2026-10-17T06:46:33.521986+00:00", "records": 181, "source_revision": "fbccd464f94fec9e284e139f97bf96d0b192f580", "previous_records_preserved": 4530, "previous_source_archives_hash_unchanged": 13, "total_records": 4711, "fixed_definitions": 3880, "parameterized_definitions": 831, "execution_runs": 0, "method": "Compared every pre-existing normalized record for exact equality and all 13 previous archive SHA-256 values after collector completion.", "snapshot": { "filename": "windowsworld.zip", "sha256": "43e9ad4a4527fdf76b23eeaf98176f84a5be0e8dd5d6f151cae69ba961ffc07d", "bytes": 140150, "files": { "Installation Guide.md": "936fa60f4dca37e984ba1e9a357c5c5abd7e2ac554748ce4ca8ab087e5258da2", "LICENSE": "a8d1d88a0f9cce3cdbff73b03a80dc5e5d69e3ba2443946c268815a550c9e0ec", "README.md": "9857bd8ae9f521f152761ae3bd85debe4e7471aad34c9eb47618332948bca6c9", "benchmark.json": "c894bbce218db9fad5f8c54b9fcdb0e77f15622efc8d14890c20aafbb4a4c551", "desktop_env/desktop_env.py": "fd5716ffe516fb9c01a749c7f6564a86708a403b68999820eba59f9b537a4a55", "hf_run.py": "dc3c4925f4a8fcf065a44438d8763867686db3bbc875500bf3ad869d9b896046", "main.py": "2e025934a656bd4bf167abe104af9be4ee18ca6a2968f773c25fbb2889a11fd1", "mm_agents/coact_agent.py": "f1eb0d94e37f737017d5400fc681126bfdb17de2deee64aa438f0606da4c2850", "requirements.txt": "d863028c0d04010eeebf131c2474ba92dd608388861c1ec8427bfb406b253c26", "show_result.py": "dc43f157648c7d58aefbce8479680852f55ae5cac4ffba3fbbc8b999263604be" } } }, "language_annotation_correction": { "checked_at": "2026-10-17T08:16:38.337622+00:00", "total_records": 4811, "unchanged_normalized_records": 4809, "language_metadata_only_changes": { "windowsworld:win_fin_l2_008": { "field": "instruction_language", "before": "en", "after": "zh-CN", "source_json_pointer": "/122", "instruction_sha256": "f2ae630fc4bee5a93176073ef0aa25ab44a31e27113cf85a2cecbdb5ec8408fa" }, "windowsworld:win_it__l1_001": { "field": "instruction_language", "before": "en", "after": "zh-CN", "source_json_pointer": "/124", "instruction_sha256": "4516e84bca1b04ffe46bf4078a083c7a58df11fc01890320e815ace286e9ec4b" } }, "all_original_fields_and_instruction_texts_unchanged": true, "all15_source_archives_unchanged": true, "before_records_sha256": "5474fe67708680a3adac6724eed982d079089861c52ea277c8c7f94873edff62", "after_records_sha256": "a2ca84410eebe71c3ea46e00449a88c25b163e7aba186542e4f84c6655bac035" } }