{"id":"scuba:admin_001_001","source_id":"scuba","original_id":"admin_001_001","kind":"benchmark_task","instruction":"Create a report using the Leads type and name it MyLeadRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nNext, add a filter so it only shows records where Lead Status is set to Open – Not Contacted.\r\nIf you don’t see the Lead Status field in the table, make sure to add it first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_001","query_templated":"Create a report with Leads type with name MyLeadRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Lead Status field equals to Open - Not Contacted. If the Lead Status field is not visible in the table, add it first.","query":"Create a report using the Leads type and name it MyLeadRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nNext, add a filter so it only shows records where Lead Status is set to Open – Not Contacted.\r\nIf you don’t see the Lead Status field in the table, make sure to add it first.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"MyLeadRPT","filter_type":"standard filter","filters":[["STATUS","equals","Open - Not Contacted"]],"field_name":"STATUS"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"MyLeadRPT","filter_type":"standard filter","filters":"Lead Status field equals to Open - Not Contacted","field_name":"Lead Status"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/0","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_001","template_name":"create_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/0"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/0"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report"}],"source_schema_notes":[],"template_registry":{"name":"create_report","original":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nCreate a report using the Leads type and name it MyLeadRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nNext, add a filter so it only shows records where Lead Status is set to Open – Not Contacted.\r\nIf you don’t see the Lead Status field in the table, make sure to add it first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_001","query_templated":"Create a report with Leads type with name MyLeadRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Lead Status field equals to Open - Not Contacted. If the Lead Status field is not visible in the table, add it first.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nCreate a report using the Leads type and name it MyLeadRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nNext, add a filter so it only shows records where Lead Status is set to Open – Not Contacted.\r\nIf you don’t see the Lead Status field in the table, make sure to add it first.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"MyLeadRPT","filter_type":"standard filter","filters":[["STATUS","equals","Open - Not Contacted"]],"field_name":"STATUS"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"MyLeadRPT","filter_type":"standard filter","filters":"Lead Status field equals to Open - Not Contacted","field_name":"Lead Status"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/0"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L191","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"93e56d4b3eb58989bb89c76195e95985c90f2906014f65216f235e7441276388"} {"id":"scuba:admin_001_002","source_id":"scuba","original_id":"admin_001_002","kind":"benchmark_task","instruction":"Now I need to create a report using the Opportunities type and name it HighValueDealsRPT.\r\nIf nothing shows up in the preview, click All Time to include results from all time.\r\nThen add a filter so it only shows records where the Amount is greater than 100,000.\r\nIf you don’t see the Amount field in the table, make sure to add it first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_002","query_templated":"Create a report with Opportunities type with name HighValueDealsRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Amount greater than 100000. If the Amount field is not visible in the table, add it first.","query":"Now I need to create a report using the Opportunities type and name it HighValueDealsRPT.\r\nIf nothing shows up in the preview, click All Time to include results from all time.\r\nThen add a filter so it only shows records where the Amount is greater than 100,000.\r\nIf you don’t see the Amount field in the table, make sure to add it first.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"HighValueDealsRPT","filter_type":"standard filter","filters":[["AMOUNT","greaterThan","\"100,000\""]],"field_name":"AMOUNT"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"HighValueDealsRPT","filter_type":"standard filter","filters":"Amount greater than 100000","field_name":"Amount"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/1","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_001","template_name":"create_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/1"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/1"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report"}],"source_schema_notes":[],"template_registry":{"name":"create_report","original":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nNow I need to create a report using the Opportunities type and name it HighValueDealsRPT.\r\nIf nothing shows up in the preview, click All Time to include results from all time.\r\nThen add a filter so it only shows records where the Amount is greater than 100,000.\r\nIf you don’t see the Amount field in the table, make sure to add it first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_002","query_templated":"Create a report with Opportunities type with name HighValueDealsRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Amount greater than 100000. If the Amount field is not visible in the table, add it first.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nNow I need to create a report using the Opportunities type and name it HighValueDealsRPT.\r\nIf nothing shows up in the preview, click All Time to include results from all time.\r\nThen add a filter so it only shows records where the Amount is greater than 100,000.\r\nIf you don’t see the Amount field in the table, make sure to add it first.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"HighValueDealsRPT","filter_type":"standard filter","filters":[["AMOUNT","greaterThan","\"100,000\""]],"field_name":"AMOUNT"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"HighValueDealsRPT","filter_type":"standard filter","filters":"Amount greater than 100000","field_name":"Amount"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/1"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L191","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b0f8979a9d3f4f2c199d11e6217bbedea76be6477cf2e0c08245cd48c50d0ba5"} {"id":"scuba:admin_001_003","source_id":"scuba","original_id":"admin_001_003","kind":"benchmark_task","instruction":"I want to create a report using the Accounts type and name it MyAccountsRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Industry is Technology and Annual Revenue is greater than 1,000,000.\r\nIf you don’t see the Industry or Annual Revenue fields in the table, add them first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_003","query_templated":"Create a report with Accounts type with name MyAccountsRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Industry equals Technology and Annual revenue is greater than 1000000. If the Industry or Annual revenue field is not visible in the table, add it first.","query":"I want to create a report using the Accounts type and name it MyAccountsRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Industry is Technology and Annual Revenue is greater than 1,000,000.\r\nIf you don’t see the Industry or Annual Revenue fields in the table, add them first.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountsRPT","filter_type":"standard filter","filters":[["INDUSTRY","equals","Technology"],["SALES","greaterThan","\"1,000,000\""]],"field_name":"INDUSTRY, SALES"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountsRPT","filter_type":"standard filter","filters":"Industry equals Technology and Annual revenue is greater than 1000000","field_name":"Industry or Annual revenue"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/2","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_001","template_name":"create_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/2"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/2"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report"}],"source_schema_notes":[],"template_registry":{"name":"create_report","original":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nI want to create a report using the Accounts type and name it MyAccountsRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Industry is Technology and Annual Revenue is greater than 1,000,000.\r\nIf you don’t see the Industry or Annual Revenue fields in the table, add them first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_003","query_templated":"Create a report with Accounts type with name MyAccountsRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Industry equals Technology and Annual revenue is greater than 1000000. If the Industry or Annual revenue field is not visible in the table, add it first.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nI want to create a report using the Accounts type and name it MyAccountsRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Industry is Technology and Annual Revenue is greater than 1,000,000.\r\nIf you don’t see the Industry or Annual Revenue fields in the table, add them first.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountsRPT","filter_type":"standard filter","filters":[["INDUSTRY","equals","Technology"],["SALES","greaterThan","\"1,000,000\""]],"field_name":"INDUSTRY, SALES"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountsRPT","filter_type":"standard filter","filters":"Industry equals Technology and Annual revenue is greater than 1000000","field_name":"Industry or Annual revenue"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/2"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L191","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"517d143625bb403b312a7da02bbb9f58160b299260579ea1c1eb7a6ba998b648"} {"id":"scuba:admin_001_004","source_id":"scuba","original_id":"admin_001_004","kind":"benchmark_task","instruction":"You need to create a report using the Tasks type and name it MyTasksRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is Completed and Priority is High.\r\nIf you don’t see the Status or Priority fields in the table, add them first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_004","query_templated":"Create a report with Tasks type with name MyTasksRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Status equals Completed and Priority equals High. If the Status or Priority field is not visible in the table, add it first.","query":"You need to create a report using the Tasks type and name it MyTasksRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is Completed and Priority is High.\r\nIf you don’t see the Status or Priority fields in the table, add them first.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Tasks and Events","report_name":"MyTasksRPT","filter_type":"standard filter","filters":[["STATUS","equals","Completed"],["PRIORITY","equals","High"]],"field_name":"STATUS, PRIORITY"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Tasks","report_name":"MyTasksRPT","filter_type":"standard filter","filters":"Status equals Completed and Priority equals High","field_name":"Status or Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/3","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_001","template_name":"create_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/3"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/3"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report"}],"source_schema_notes":[],"template_registry":{"name":"create_report","original":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nYou need to create a report using the Tasks type and name it MyTasksRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is Completed and Priority is High.\r\nIf you don’t see the Status or Priority fields in the table, add them first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_004","query_templated":"Create a report with Tasks type with name MyTasksRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Status equals Completed and Priority equals High. If the Status or Priority field is not visible in the table, add it first.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nYou need to create a report using the Tasks type and name it MyTasksRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is Completed and Priority is High.\r\nIf you don’t see the Status or Priority fields in the table, add them first.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Tasks and Events","report_name":"MyTasksRPT","filter_type":"standard filter","filters":[["STATUS","equals","Completed"],["PRIORITY","equals","High"]],"field_name":"STATUS, PRIORITY"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Tasks","report_name":"MyTasksRPT","filter_type":"standard filter","filters":"Status equals Completed and Priority equals High","field_name":"Status or Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/3"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L191","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"30a8f2b876c0e9451d9406ca45a8b37f908b82cf4da34335dcd3ba4c4ebb8c9e"} {"id":"scuba:admin_001_005","source_id":"scuba","original_id":"admin_001_005","kind":"benchmark_task","instruction":"Please create a report using the Cases type and name it OpenCasesRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is New and Origin is Phone.\r\nIf you don’t see the Status or Origin fields in the table, add them first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_005","query_templated":"Create a report with Cases type with name OpenCasesRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Status equals New and Origin equals Phone. If the Status or Origin field is not visible in the table, add it first.","query":"Please create a report using the Cases type and name it OpenCasesRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is New and Origin is Phone.\r\nIf you don’t see the Status or Origin fields in the table, add them first.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Cases","report_name":"OpenCasesRPT","filter_type":"standard filter","filters":[["STATUS","equals","New"],["ORIGIN","equals","Phone"]],"field_name":"STATUS, ORIGIN"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"OpenCasesRPT","filter_type":"standard filter","filters":"Status equals New and Origin equals Phone","field_name":"Status or Origin"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/4","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_001","template_name":"create_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/4"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/4"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report"}],"source_schema_notes":[],"template_registry":{"name":"create_report","original":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nPlease create a report using the Cases type and name it OpenCasesRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is New and Origin is Phone.\r\nIf you don’t see the Status or Origin fields in the table, add them first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_001_005","query_templated":"Create a report with Cases type with name OpenCasesRPT. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose Status equals New and Origin equals Phone. If the Status or Origin field is not visible in the table, add it first.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory.\n\n## Your ultimate task\n```\nPlease create a report using the Cases type and name it OpenCasesRPT.\r\nIf no data shows up in the preview, click All Time so it includes results from all time.\r\nThen add filters so it only shows records where Status is New and Origin is Phone.\r\nIf you don’t see the Status or Origin fields in the table, add them first.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Cases","report_name":"OpenCasesRPT","filter_type":"standard filter","filters":[["STATUS","equals","New"],["ORIGIN","equals","Phone"]],"field_name":"STATUS, ORIGIN"},"difficulty":"medium","query_template_name":"create_report","query_template_metadata":{"template_id":"admin_001","template_string":"Create a report with {object_name} type with name {report_name}. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a {filter_type} to select records whose {filters}. If the {field_name} field is not visible in the table, add it first.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"OpenCasesRPT","filter_type":"standard filter","filters":"Status equals New and Origin equals Phone","field_name":"Status or Origin"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with [Leads] type with name [MyLeadRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a [standard filter] to select records whose [Lead Status] field [equals to] [Open - Not Contacted]. If the [Lead Status] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In Report Type, search for “leads,” select Leads, and click Start Report.\n- Clear default filters (Email, Street, Rating) if present.\n- Add the Lead Status column:\n - Use Add Column search to find “lead status” and add it.\n- Open Filters panel and add a standard filter:\n - Search/select Lead Status.\n - Set equals to “Open - Not Contacted” and Apply.\n- Click Refresh to update the preview.\n- Save the report:\n - Enter report name “MyLeadRPT” and click Save.\nSample 2:\nTASK:Create a report with [Opportunities] type named [HighValueDealsRPT]. If you don’t see any data in the preview area, click the All Time link to apply a standard filter that shows results for all time. Add a standard filter to select records whose [Amount] field is greater than [100,000]. If the [Amount] field is not visible in the table, add it first.\nEXPERIENCE:\nSuccessful plan to create the report:\n- Log in.\n- Open App Launcher, search for “reports,” and open Reports.\n- Click New Report.\n- In the report type selector, choose All, search for “opportunities,” select Opportunities, and click Start Report.\n- Open the Filters panel, add a filter for Amount:\n - Select Amount, set Operator to “greater than,” enter 100000, and Apply.\n- Click Save, set Report Name to “HighValueDealsRPT,” and Save again.\n\nNotes:\n- No hot-keys were used in the trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/4"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L191","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9ed2cae00f9561752054d81f977e49183bbfad333a3e206eb0b0264939d624b5"} {"id":"scuba:admin_002_001","source_id":"scuba","original_id":"admin_002_001","kind":"benchmark_task","instruction":"Let’s go ahead and build a report using the Accounts type and call it MyAccountRPT.\r\nNext, add a cross filter that brings in the Opportunities object.\r\nFrom there, apply a subfilter so it only shows opportunities where the Stage is Prospecting and Need Analysis.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_001","query_templated":"Create a report with the Accounts type with name MyAccountRPT. Add a cross filter with the Opportunities object. Add a subfilter on stage field of the Opportunities object, which is set to equal to Prospecting and Need Analysis..","query":"Let’s go ahead and build a report using the Accounts type and call it MyAccountRPT.\r\nNext, add a cross filter that brings in the Opportunities object.\r\nFrom there, apply a subfilter so it only shows opportunities where the Stage is Prospecting and Need Analysis.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountRPT","cross_object_name":"Opportunity","filters":[["StageName","equals","Prospecting,Needs Analysis"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountRPT","cross_object_name":"Opportunities","filters":"Add a subfilter on stage field of the Opportunities object, which is set to equal to Prospecting and Need Analysis."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/5","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"easy","template_id":"admin_002","template_name":"create_report_with_cross_filter","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/5"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/5"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_with_cross_filter"}],"source_schema_notes":[],"template_registry":{"name":"create_report_with_cross_filter","original":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s go ahead and build a report using the Accounts type and call it MyAccountRPT.\r\nNext, add a cross filter that brings in the Opportunities object.\r\nFrom there, apply a subfilter so it only shows opportunities where the Stage is Prospecting and Need Analysis.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_001","query_templated":"Create a report with the Accounts type with name MyAccountRPT. Add a cross filter with the Opportunities object. Add a subfilter on stage field of the Opportunities object, which is set to equal to Prospecting and Need Analysis..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s go ahead and build a report using the Accounts type and call it MyAccountRPT.\r\nNext, add a cross filter that brings in the Opportunities object.\r\nFrom there, apply a subfilter so it only shows opportunities where the Stage is Prospecting and Need Analysis.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountRPT","cross_object_name":"Opportunity","filters":[["StageName","equals","Prospecting,Needs Analysis"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountRPT","cross_object_name":"Opportunities","filters":"Add a subfilter on stage field of the Opportunities object, which is set to equal to Prospecting and Need Analysis."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/5"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L624","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_with_cross_filter","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6c07fcb64cf25abfb7fa3c552d47a9b52e0ecdda6f6319190291b7e23de80f83"} {"id":"scuba:admin_002_002","source_id":"scuba","original_id":"admin_002_002","kind":"benchmark_task","instruction":"Start by creating a report using the Leads type and name it QualifiedLeadsRPT.\r\nAfter that, add a cross filter that pulls in the Activities object.\r\nThen, set up two subfilters: one where Status is Completed, and another where Subject contains the word Demo.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_002","query_templated":"Create a report with the Leads type with name QualifiedLeadsRPT. Add a cross filter with the Activities object. Add subfilters on the Status field of the Activities object set to equals Completed, and on the Subject field set to contains Demo..","query":"Start by creating a report using the Leads type and name it QualifiedLeadsRPT.\r\nAfter that, add a cross filter that pulls in the Activities object.\r\nThen, set up two subfilters: one where Status is Completed, and another where Subject contains the word Demo.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"QualifiedLeadsRPT","cross_object_name":"Activity","filters":[["Status","equals","Completed"],["Subject","contains","Demo"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"QualifiedLeadsRPT","cross_object_name":"Activities","filters":"Add subfilters on the Status field of the Activities object set to equals Completed, and on the Subject field set to contains Demo."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/6","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"easy","template_id":"admin_002","template_name":"create_report_with_cross_filter","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/6"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/6"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_with_cross_filter"}],"source_schema_notes":[],"template_registry":{"name":"create_report_with_cross_filter","original":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nStart by creating a report using the Leads type and name it QualifiedLeadsRPT.\r\nAfter that, add a cross filter that pulls in the Activities object.\r\nThen, set up two subfilters: one where Status is Completed, and another where Subject contains the word Demo.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_002","query_templated":"Create a report with the Leads type with name QualifiedLeadsRPT. Add a cross filter with the Activities object. Add subfilters on the Status field of the Activities object set to equals Completed, and on the Subject field set to contains Demo..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nStart by creating a report using the Leads type and name it QualifiedLeadsRPT.\r\nAfter that, add a cross filter that pulls in the Activities object.\r\nThen, set up two subfilters: one where Status is Completed, and another where Subject contains the word Demo.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"QualifiedLeadsRPT","cross_object_name":"Activity","filters":[["Status","equals","Completed"],["Subject","contains","Demo"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"QualifiedLeadsRPT","cross_object_name":"Activities","filters":"Add subfilters on the Status field of the Activities object set to equals Completed, and on the Subject field set to contains Demo."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/6"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L624","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_with_cross_filter","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f4717e9bc5dbc38385ae61c547dbdeddab5f865bfcf60763b4e9f0a2f259ccfb"} {"id":"scuba:admin_002_003","source_id":"scuba","original_id":"admin_002_003","kind":"benchmark_task","instruction":"Go ahead and create a report using the Cases type and call it CasesWithAnimalCommentsRPT.\r\nOnce that’s set up, add a cross filter that brings in the Case Comments object.\r\nFor the subfilters, make sure Created Date is set to less than or equal to today, and that the Body field includes the word “animal”.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_003","query_templated":"Create a report with the Cases type with name CasesWithAnimalCommentsRPT. Add a cross filter with the Case Comments object. Add subfilters on the Created Date field of the Case Comments object set to less than or equal to today, and on the body field set to contains “animal”..","query":"Go ahead and create a report using the Cases type and call it CasesWithAnimalCommentsRPT.\r\nOnce that’s set up, add a cross filter that brings in the Case Comments object.\r\nFor the subfilters, make sure Created Date is set to less than or equal to today, and that the Body field includes the word “animal”.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Cases","report_name":"CasesWithAnimalCommentsRPT","cross_object_name":"CaseComment","filters":[["CreatedDate","lessOrEqual","TODAY"],["CommentBody","contains","animal"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CasesWithAnimalCommentsRPT","cross_object_name":"Case Comments","filters":"Add subfilters on the Created Date field of the Case Comments object set to less than or equal to today, and on the body field set to contains “animal”."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/7","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"easy","template_id":"admin_002","template_name":"create_report_with_cross_filter","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/7"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/7"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_with_cross_filter"}],"source_schema_notes":[],"template_registry":{"name":"create_report_with_cross_filter","original":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGo ahead and create a report using the Cases type and call it CasesWithAnimalCommentsRPT.\r\nOnce that’s set up, add a cross filter that brings in the Case Comments object.\r\nFor the subfilters, make sure Created Date is set to less than or equal to today, and that the Body field includes the word “animal”.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_003","query_templated":"Create a report with the Cases type with name CasesWithAnimalCommentsRPT. Add a cross filter with the Case Comments object. Add subfilters on the Created Date field of the Case Comments object set to less than or equal to today, and on the body field set to contains “animal”..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGo ahead and create a report using the Cases type and call it CasesWithAnimalCommentsRPT.\r\nOnce that’s set up, add a cross filter that brings in the Case Comments object.\r\nFor the subfilters, make sure Created Date is set to less than or equal to today, and that the Body field includes the word “animal”.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Cases","report_name":"CasesWithAnimalCommentsRPT","cross_object_name":"CaseComment","filters":[["CreatedDate","lessOrEqual","TODAY"],["CommentBody","contains","animal"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CasesWithAnimalCommentsRPT","cross_object_name":"Case Comments","filters":"Add subfilters on the Created Date field of the Case Comments object set to less than or equal to today, and on the body field set to contains “animal”."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/7"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L624","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_with_cross_filter","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"be1ebf22e7fc521c1e2eafafdf526c2227b0d248e972ca470af6cd39909caed7"} {"id":"scuba:admin_002_004","source_id":"scuba","original_id":"admin_002_004","kind":"benchmark_task","instruction":"Could you please create a report using the Leads type and name it LeadTerritoriesEngagementRPT?\r\nAfter that, add a cross filter that includes the Territories object.\r\nFinally, add a subfilter so that the Method field of the territory is set to not equal to Territory Manual.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_004","query_templated":"Create a report with the Leads type with name LeadTerritoriesEngagementRPT. Add a cross filter with the Territories object. Add subfilters on the Method field of the Territory object set to not equal to Territory Manual.","query":"Could you please create a report using the Leads type and name it LeadTerritoriesEngagementRPT?\r\nAfter that, add a cross filter that includes the Territories object.\r\nFinally, add a subfilter so that the Method field of the territory is set to not equal to Territory Manual.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"LeadTerritoriesEngagementRPT","cross_object_name":"ObjectTerritory2Association-Territory2","filters":[["AssociationCause","notEqual","Territory Manual"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"LeadTerritoriesEngagementRPT","cross_object_name":"Territories","filters":"Add subfilters on the Method field of the Territory object set to not equal to Territory Manual"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/8","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"easy","template_id":"admin_002","template_name":"create_report_with_cross_filter","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/8"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/8"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_with_cross_filter"}],"source_schema_notes":[],"template_registry":{"name":"create_report_with_cross_filter","original":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you please create a report using the Leads type and name it LeadTerritoriesEngagementRPT?\r\nAfter that, add a cross filter that includes the Territories object.\r\nFinally, add a subfilter so that the Method field of the territory is set to not equal to Territory Manual.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_004","query_templated":"Create a report with the Leads type with name LeadTerritoriesEngagementRPT. Add a cross filter with the Territories object. Add subfilters on the Method field of the Territory object set to not equal to Territory Manual.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you please create a report using the Leads type and name it LeadTerritoriesEngagementRPT?\r\nAfter that, add a cross filter that includes the Territories object.\r\nFinally, add a subfilter so that the Method field of the territory is set to not equal to Territory Manual.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Leads","report_name":"LeadTerritoriesEngagementRPT","cross_object_name":"ObjectTerritory2Association-Territory2","filters":[["AssociationCause","notEqual","Territory Manual"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"LeadTerritoriesEngagementRPT","cross_object_name":"Territories","filters":"Add subfilters on the Method field of the Territory object set to not equal to Territory Manual"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/8"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L624","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_with_cross_filter","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"08dab60c23d43a0662a3b72fd4699eb279be9ef57b3a3a7fcc91412bfbfb097a"} {"id":"scuba:admin_002_005","source_id":"scuba","original_id":"admin_002_005","kind":"benchmark_task","instruction":"1. Create a report using the Opportunities type and give it the name Competitors.\r\n\r\n2. Add a cross filter that includes the Products object.\r\n\r\n3. Then, add a subfilter so that within the Competitor object, the Name field contains CRM.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_005","query_templated":"Create a report with the Opportunities type with name Competitors. Add a cross filter with the Products object. Add subfilters on the Name field of the Competitor object that contains CRM.","query":"1. Create a report using the Opportunities type and give it the name Competitors.\r\n\r\n2. Add a cross filter that includes the Products object.\r\n\r\n3. Then, add a subfilter so that within the Competitor object, the Name field contains CRM.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"Competitors","cross_object_name":"OpportunityCompetitor","filters":[["CompetitorName","contains","CRM"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"Competitors","cross_object_name":"Products","filters":"Add subfilters on the Name field of the Competitor object that contains CRM"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/9","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"easy","template_id":"admin_002","template_name":"create_report_with_cross_filter","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/9"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/9"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_with_cross_filter"}],"source_schema_notes":[],"template_registry":{"name":"create_report_with_cross_filter","original":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_with_cross_filter","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\n1. Create a report using the Opportunities type and give it the name Competitors.\r\n\r\n2. Add a cross filter that includes the Products object.\r\n\r\n3. Then, add a subfilter so that within the Competitor object, the Name field contains CRM.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_002_005","query_templated":"Create a report with the Opportunities type with name Competitors. Add a cross filter with the Products object. Add subfilters on the Name field of the Competitor object that contains CRM.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\n1. Create a report using the Opportunities type and give it the name Competitors.\r\n\r\n2. Add a cross filter that includes the Products object.\r\n\r\n3. Then, add a subfilter so that within the Competitor object, the Name field contains CRM.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"Competitors","cross_object_name":"OpportunityCompetitor","filters":[["CompetitorName","contains","CRM"]]},"difficulty":"easy","query_template_name":"create_report_with_cross_filter","query_template_metadata":{"template_id":"admin_002","template_string":"Create a report with the {object_name} type with name {report_name}. Add a cross filter with the {cross_object_name} object. {filters}.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"Competitors","cross_object_name":"Products","filters":"Add subfilters on the Name field of the Competitor object that contains CRM"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the [Accounts] type with name [MyAccountRPT]. Add a [cross filter] with the [Opportunities] object. Add a subfilter on [stage] field of the [Opportunities] object, which is set to [equal to] [Prospecting and Need Analysis.]\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Reports,” and open Reports.\n- Click New Report, select the “Accounts” report type, and click Start Report.\n- Open the Filters panel.\n- Add Cross Filter: Accounts with Opportunities, then Apply.\n- Add an opportunity subfilter: choose Stage, select Prospecting and Needs Analysis, then Apply.\n- Click Save & Run, set Report Name to “MyAccountRPT,” and click Save.\nSample 2:\nTASK:Create a report with the [Leads] type named [QualifiedLeadsRPT]. Add a [cross filter] with the [Activities] object. Add subfilters on the [Status] field of the [Activities] object set to [equals Completed], and on the [Subject] field set to [contains Demo].\nEXPERIENCE:\nSuccessful plan to create the report:\n\n- Open Reports:\n - Open App Launcher, search for “Reports,” and open the Reports app/page.\n - Click New Report.\n - Select Leads (under the Leads category) and click Start Report.\n\n- Add cross filter on Activities:\n - Open the Filters pane.\n - Open the Filters dropdown and choose Add Cross Filter.\n - Ensure the related object is Activities and apply.\n\n- Add Activities subfilters:\n - Click Add Activities filter, choose Status, set to equals Completed, and apply.\n - Click Add Activities filter, choose Subject, change operator to contains, enter Demo, and apply.\n\n- Save the report:\n - Click Save & Run.\n - Set Report Name to QualifiedLeadsRPT.\n - Click Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/9"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L624","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_with_cross_filter","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"63d6f4073c95e213b4de4a20e3a5c98ac8c9f3f60f0d603e45314c4ec427f78f"} {"id":"scuba:admin_003_001","source_id":"scuba","original_id":"admin_003_001","kind":"benchmark_task","instruction":"Please spin up a Cases report named UnimportantCasesRPT. Add these standard filters, numbered as follows: 1) Status equals New 2) Date/Time Opened is greater than Last Week 3) Priority equals Low 4) Escalated equals true. Then apply this filter logic: (1 OR 2) AND 3 AND NOT 4. Save the report in the Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_001","query_templated":"Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"Please spin up a Cases report named UnimportantCasesRPT. Add these standard filters, numbered as follows: 1) Status equals New 2) Date/Time Opened is greater than Last Week 3) Priority equals Low 4) Escalated equals true. Then apply this filter logic: (1 OR 2) AND 3 AND NOT 4. Save the report in the Public Reports.","human_review_comment":"delete folder","ground_truth_dict":{"object_name":"Cases","report_name":"UnimportantCasesRPT","field1":"STATUS","operator1":"equals","value1":"New","field2":"CREATED_DATE","operator2":"greaterThan","value2":"LAST WEEK","field3":"PRIORITY","operator3":"equals","value3":"Low","field4":"ESCALATION_STATE","operator4":"equals","value4":"True"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"UnimportantCasesRPT","field1":"Status","operator1":"equals","value1":"New","field2":"Date/Time Opened","operator2":"greater than","value2":"Last Week","field3":"Priority","operator3":"equals","value3":"Low","field4":"Escalated","operator4":"equals","value4":"true"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/10","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"hard","template_id":"admin_003","template_name":"create_report_chain_filters","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/10"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/10"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_chain_filters"}],"source_schema_notes":[],"template_registry":{"name":"create_report_chain_filters","original":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nPlease spin up a Cases report named UnimportantCasesRPT. Add these standard filters, numbered as follows: 1) Status equals New 2) Date/Time Opened is greater than Last Week 3) Priority equals Low 4) Escalated equals true. Then apply this filter logic: (1 OR 2) AND 3 AND NOT 4. Save the report in the Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_001","query_templated":"Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nPlease spin up a Cases report named UnimportantCasesRPT. Add these standard filters, numbered as follows: 1) Status equals New 2) Date/Time Opened is greater than Last Week 3) Priority equals Low 4) Escalated equals true. Then apply this filter logic: (1 OR 2) AND 3 AND NOT 4. Save the report in the Public Reports.\n```","human_review_comment":"delete folder","ground_truth_dict":{"object_name":"Cases","report_name":"UnimportantCasesRPT","field1":"STATUS","operator1":"equals","value1":"New","field2":"CREATED_DATE","operator2":"greaterThan","value2":"LAST WEEK","field3":"PRIORITY","operator3":"equals","value3":"Low","field4":"ESCALATION_STATE","operator4":"equals","value4":"True"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"UnimportantCasesRPT","field1":"Status","operator1":"equals","value1":"New","field2":"Date/Time Opened","operator2":"greater than","value2":"Last Week","field3":"Priority","operator3":"equals","value3":"Low","field4":"Escalated","operator4":"equals","value4":"true"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/10"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L474","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_chain_filters","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e0652d37d2ba3d207bf1848913a4628a682d392dcebb55b68492f59c7c39a8c8"} {"id":"scuba:admin_003_002","source_id":"scuba","original_id":"admin_003_002","kind":"benchmark_task","instruction":"I need an Opportunities report called ClosedOldBusinessOpportunitiesRPT. Use standard filters with this numbering: 1. Stage = Closed Won 2. Stage = Closed Lost 3. Amount >= 5000 4. Type = New Customer. Combine them using filter logic: (1 OR 2) AND 3 AND NOT 4. Save it to the Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_002","query_templated":"Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"I need an Opportunities report called ClosedOldBusinessOpportunitiesRPT. Use standard filters with this numbering: 1. Stage = Closed Won 2. Stage = Closed Lost 3. Amount >= 5000 4. Type = New Customer. Combine them using filter logic: (1 OR 2) AND 3 AND NOT 4. Save it to the Public Reports.","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"ClosedOldBusinessOpportunitiesRPT","field1":"STAGE_NAME","operator1":"equals","value1":"Closed Won","field2":"STAGE_NAME","operator2":"equals","value2":"Closed Lost","field3":"AMOUNT","operator3":"greaterOrEqual","value3":"\"5,000\"","field4":"TYPE","operator4":"equals","value4":"New Customer"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"ClosedOldBusinessOpportunitiesRPT","field1":"Stage","operator1":"equals","value1":"Closed Won","field2":"Stage","operator2":"equals","value2":"Closed Lost","field3":"Amount","operator3":"Greater Than or Equal to","value3":5000,"field4":"Type","operator4":"equals","value4":"New Customer"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/11","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"hard","template_id":"admin_003","template_name":"create_report_chain_filters","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/11"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/11"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_chain_filters"}],"source_schema_notes":[],"template_registry":{"name":"create_report_chain_filters","original":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nI need an Opportunities report called ClosedOldBusinessOpportunitiesRPT. Use standard filters with this numbering: 1. Stage = Closed Won 2. Stage = Closed Lost 3. Amount >= 5000 4. Type = New Customer. Combine them using filter logic: (1 OR 2) AND 3 AND NOT 4. Save it to the Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_002","query_templated":"Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nI need an Opportunities report called ClosedOldBusinessOpportunitiesRPT. Use standard filters with this numbering: 1. Stage = Closed Won 2. Stage = Closed Lost 3. Amount >= 5000 4. Type = New Customer. Combine them using filter logic: (1 OR 2) AND 3 AND NOT 4. Save it to the Public Reports.\n```","human_review_comment":"rejected since it is the same. Rewriten","ground_truth_dict":{"object_name":"Opportunities","report_name":"ClosedOldBusinessOpportunitiesRPT","field1":"STAGE_NAME","operator1":"equals","value1":"Closed Won","field2":"STAGE_NAME","operator2":"equals","value2":"Closed Lost","field3":"AMOUNT","operator3":"greaterOrEqual","value3":"\"5,000\"","field4":"TYPE","operator4":"equals","value4":"New Customer"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"ClosedOldBusinessOpportunitiesRPT","field1":"Stage","operator1":"equals","value1":"Closed Won","field2":"Stage","operator2":"equals","value2":"Closed Lost","field3":"Amount","operator3":"Greater Than or Equal to","value3":5000,"field4":"Type","operator4":"equals","value4":"New Customer"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/11"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L474","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_chain_filters","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"73b9aa2f1ac0da272f1bfb5cb5522d55b4ad01de01150835bc2da85e49c0d4fb"} {"id":"scuba:admin_003_003","source_id":"scuba","original_id":"admin_003_003","kind":"benchmark_task","instruction":"Build an Accounts report titled UnimportantAccountsRPT and wire up these standard filters (keep the numbering): 1) Rating not equals Hot 2) Annual Revenue less than 10000 3) Industry not equals Technology 4) Industry equals Other Now set the filter logic to: (1 OR 2) AND 3 AND NOT 4. Please save it under Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_003","query_templated":"Create a report with the Accounts type with name UnimportantAccountsRPT. Add following standard filters: [1. set Rating field not equals Hot 2. Set Annual Revenue field less than 10000 3. Set Industry field notequals Technology 4. set Industry field equals Other.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"Build an Accounts report titled UnimportantAccountsRPT and wire up these standard filters (keep the numbering): 1) Rating not equals Hot 2) Annual Revenue less than 10000 3) Industry not equals Technology 4) Industry equals Other Now set the filter logic to: (1 OR 2) AND 3 AND NOT 4. Please save it under Public Reports.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Accounts","report_name":"UnimportantAccountsRPT","field1":"RATING","operator1":"notEqual","value1":"Hot","field2":"SALES","operator2":"lessThan","value2":"\"10,000\"","field3":"INDUSTRY","operator3":"notEqual","value3":"Technology","field4":"INDUSTRY","operator4":"equals","value4":"Other"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"UnimportantAccountsRPT","field1":"Rating","operator1":"not equals","value1":"Hot","field2":"Annual Revenue","operator2":"less than","value2":10000,"field3":"Industry","operator3":"notequals","value3":"Technology","field4":"Industry","operator4":"equals","value4":"Other"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/12","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"hard","template_id":"admin_003","template_name":"create_report_chain_filters","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/12"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/12"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_chain_filters"}],"source_schema_notes":[],"template_registry":{"name":"create_report_chain_filters","original":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nBuild an Accounts report titled UnimportantAccountsRPT and wire up these standard filters (keep the numbering): 1) Rating not equals Hot 2) Annual Revenue less than 10000 3) Industry not equals Technology 4) Industry equals Other Now set the filter logic to: (1 OR 2) AND 3 AND NOT 4. Please save it under Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_003","query_templated":"Create a report with the Accounts type with name UnimportantAccountsRPT. Add following standard filters: [1. set Rating field not equals Hot 2. Set Annual Revenue field less than 10000 3. Set Industry field notequals Technology 4. set Industry field equals Other.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nBuild an Accounts report titled UnimportantAccountsRPT and wire up these standard filters (keep the numbering): 1) Rating not equals Hot 2) Annual Revenue less than 10000 3) Industry not equals Technology 4) Industry equals Other Now set the filter logic to: (1 OR 2) AND 3 AND NOT 4. Please save it under Public Reports.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Accounts","report_name":"UnimportantAccountsRPT","field1":"RATING","operator1":"notEqual","value1":"Hot","field2":"SALES","operator2":"lessThan","value2":"\"10,000\"","field3":"INDUSTRY","operator3":"notEqual","value3":"Technology","field4":"INDUSTRY","operator4":"equals","value4":"Other"},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"UnimportantAccountsRPT","field1":"Rating","operator1":"not equals","value1":"Hot","field2":"Annual Revenue","operator2":"less than","value2":10000,"field3":"Industry","operator3":"notequals","value3":"Technology","field4":"Industry","operator4":"equals","value4":"Other"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/12"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L474","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_chain_filters","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5d5c4249dc90e3688a45f0c897f121d5e46ec9b311664f538965849a8c47779e"} {"id":"scuba:admin_003_004","source_id":"scuba","original_id":"admin_003_004","kind":"benchmark_task","instruction":"Create a Products report named ActiveStaleProductsRPT. Add the following standard filters, labeled 1 through 4: 1) LastModified Date less than last year 2) Description contains old 3) Active equals true 4) Product Family equals empty value. Use this filter logic: (1 OR 2) AND 3 AND NOT 4. Store it in the Public Reports folder.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_004","query_templated":"Create a report with the Products type with name ActiveStaleProductsRPT. Add following standard filters: [1. set LastModified Date field less than last year 2. Set Description field contains old 3. Set Active field equals true 4. set Product Family field equals empty value.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"Create a Products report named ActiveStaleProductsRPT. Add the following standard filters, labeled 1 through 4: 1) LastModified Date less than last year 2) Description contains old 3) Active equals true 4) Product Family equals empty value. Use this filter logic: (1 OR 2) AND 3 AND NOT 4. Store it in the Public Reports folder.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Products","report_name":"ActiveStaleProductsRPT","field1":"LAST_UPDATE","operator1":"lessThan","value1":"LAST YEAR","field2":"DESCRIPTION","operator2":"contains","value2":"old","field3":"ACTIVE","operator3":"equals","value3":"True","field4":"FAMILY","operator4":"equals","value4":""},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Products","report_name":"ActiveStaleProductsRPT","field1":"LastModified Date","operator1":"less than","value1":"last year","field2":"Description","operator2":"contains","value2":"old","field3":"Active","operator3":"equals","value3":"true","field4":"Product Family","operator4":"equals","value4":"empty value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/13","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"hard","template_id":"admin_003","template_name":"create_report_chain_filters","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/13"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/13"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_chain_filters"}],"source_schema_notes":[],"template_registry":{"name":"create_report_chain_filters","original":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nCreate a Products report named ActiveStaleProductsRPT. Add the following standard filters, labeled 1 through 4: 1) LastModified Date less than last year 2) Description contains old 3) Active equals true 4) Product Family equals empty value. Use this filter logic: (1 OR 2) AND 3 AND NOT 4. Store it in the Public Reports folder.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_004","query_templated":"Create a report with the Products type with name ActiveStaleProductsRPT. Add following standard filters: [1. set LastModified Date field less than last year 2. Set Description field contains old 3. Set Active field equals true 4. set Product Family field equals empty value.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nCreate a Products report named ActiveStaleProductsRPT. Add the following standard filters, labeled 1 through 4: 1) LastModified Date less than last year 2) Description contains old 3) Active equals true 4) Product Family equals empty value. Use this filter logic: (1 OR 2) AND 3 AND NOT 4. Store it in the Public Reports folder.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Products","report_name":"ActiveStaleProductsRPT","field1":"LAST_UPDATE","operator1":"lessThan","value1":"LAST YEAR","field2":"DESCRIPTION","operator2":"contains","value2":"old","field3":"ACTIVE","operator3":"equals","value3":"True","field4":"FAMILY","operator4":"equals","value4":""},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Products","report_name":"ActiveStaleProductsRPT","field1":"LastModified Date","operator1":"less than","value1":"last year","field2":"Description","operator2":"contains","value2":"old","field3":"Active","operator3":"equals","value3":"true","field4":"Product Family","operator4":"equals","value4":"empty value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/13"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L474","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_chain_filters","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f15ef9d3c33b747d23c1260b480ed313e8f1dc426065e5b2035df2152f83732e"} {"id":"scuba:admin_003_005","source_id":"scuba","original_id":"admin_003_005","kind":"benchmark_task","instruction":"Could you set up a Leads report called QualifiedOnlineLeadsRPT? Add these standard filters and keep the numbering: 1) Lead Source equals Web 2) Lead Source equals Other 3) Lead Status equals Working - Contacted 4) Company / Account equals empty value. Apply filter logic: (1 OR 2) AND 3 AND NOT 4. Save the finished report to Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_005","query_templated":"Create a report with the Leads type with name QualifiedOnlineLeadsRPT. Add following standard filters: [1. set Lead Source field equals Web 2. Set Lead Source field equals Other 3. Set Lead Status field equals Working - Contacted 4. set Company / Account field equals empty value.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"Could you set up a Leads report called QualifiedOnlineLeadsRPT? Add these standard filters and keep the numbering: 1) Lead Source equals Web 2) Lead Source equals Other 3) Lead Status equals Working - Contacted 4) Company / Account equals empty value. Apply filter logic: (1 OR 2) AND 3 AND NOT 4. Save the finished report to Public Reports.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Leads","report_name":"QualifiedOnlineLeadsRPT","field1":"LEAD_SOURCE","operator1":"equals","value1":"Web","field2":"LEAD_SOURCE","operator2":"equals","value2":"Other","field3":"STATUS","operator3":"equals","value3":"Working - Contacted","field4":"COMPANY","operator4":"equals","value4":""},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"QualifiedOnlineLeadsRPT","field1":"Lead Source","operator1":"equals","value1":"Web","field2":"Lead Source","operator2":"equals","value2":"Other","field3":"Lead Status","operator3":"equals","value3":"Working - Contacted","field4":"Company / Account","operator4":"equals","value4":"empty value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/14","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"hard","template_id":"admin_003","template_name":"create_report_chain_filters","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/14"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/14"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_report_chain_filters"}],"source_schema_notes":[],"template_registry":{"name":"create_report_chain_filters","original":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_report_chain_filters","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nCould you set up a Leads report called QualifiedOnlineLeadsRPT? Add these standard filters and keep the numbering: 1) Lead Source equals Web 2) Lead Source equals Other 3) Lead Status equals Working - Contacted 4) Company / Account equals empty value. Apply filter logic: (1 OR 2) AND 3 AND NOT 4. Save the finished report to Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_003_005","query_templated":"Create a report with the Leads type with name QualifiedOnlineLeadsRPT. Add following standard filters: [1. set Lead Source field equals Web 2. Set Lead Source field equals Other 3. Set Lead Status field equals Working - Contacted 4. set Company / Account field equals empty value.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save.\n\n## Your ultimate task\n```\nCould you set up a Leads report called QualifiedOnlineLeadsRPT? Add these standard filters and keep the numbering: 1) Lead Source equals Web 2) Lead Source equals Other 3) Lead Status equals Working - Contacted 4) Company / Account equals empty value. Apply filter logic: (1 OR 2) AND 3 AND NOT 4. Save the finished report to Public Reports.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Leads","report_name":"QualifiedOnlineLeadsRPT","field1":"LEAD_SOURCE","operator1":"equals","value1":"Web","field2":"LEAD_SOURCE","operator2":"equals","value2":"Other","field3":"STATUS","operator3":"equals","value3":"Working - Contacted","field4":"COMPANY","operator4":"equals","value4":""},"difficulty":"hard","query_template_name":"create_report_chain_filters","query_template_metadata":{"template_id":"admin_003","template_string":"Create a report with the {object_name} type with name {report_name}. Add following standard filters: [1. set {field1} field {operator1} {value1} 2. Set {field2} field {operator2} {value2} 3. Set {field3} field {operator3} {value3} 4. set {field4} field {operator4} {value4}.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"QualifiedOnlineLeadsRPT","field1":"Lead Source","operator1":"equals","value1":"Web","field2":"Lead Source","operator2":"equals","value2":"Other","field3":"Lead Status","operator3":"equals","value3":"Working - Contacted","field4":"Company / Account","operator4":"equals","value4":"empty value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a report with the Cases type with name UnimportantCasesRPT. Add following standard filters: [1. set Status field equals New 2. Set Date/Time Opened field greater than Last Week 3. Set Priority field equals Low 4. set Escalated field equals true.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open the App Launcher, search for “Reports,” and open the Reports app.\n- Click “New Report,” select “Cases,” and click “Start Report.”\n- Open the Filters panel and add these filters:\n 1) Status equals “New”\n 2) Date/Time Opened greater than “Last Week” (Use Relative Date: Last Week)\n 3) Priority equals “Low”\n 4) Escalated equals “True”\n- Add filter logic and set it to: (1 OR 2) AND 3 NOT 4\n- Click “Save & Run,” choose the “Public Reports” folder, name the report “UnimportantCasesRPT,” and save.\nSample 2:\nTASK:Create a report with the Opportunities type with name ClosedOldBusinessOpportunitiesRPT. Add following standard filters: [1. set Stage field equals Closed Won 2. Set Stage field equals Closed Lost 3. Set Amount field Greater Than or Equal to 5000 4. set Type field equals New Customer.] Chain the filters with the logic filter with the formula [(1 OR 2) AND 3 NOT 4]. Save it to the public report.\nEXPERIENCE:\nSuccessful plan:\n\n- Open Reports: Use App Launcher, search for “Reports,” and open it. Click New Report, select “Opportunities,” then Start Report.\n- Add filters in the Filters panel:\n 1) Stage equals “Closed Won” and Apply.\n 2) Stage equals “Closed Lost” and Apply.\n 3) Amount operator “Greater or equal” with value “5000” and Apply.\n 4) Type equals “New Customer” and Apply.\n- Add filter logic: Open Filters dropdown > Add Filter Logic, enter “(1 OR 2) AND 3 AND NOT 4” and Apply.\n- Save to public folder: Click Save & Run, set Report Name to “ClosedOldBusinessOpportunitiesRPT,” choose “Public Reports” as the folder, and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/14"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L474","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_report_chain_filters","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9024f8413c6fc11758629bf86fd2f088fe36392d16efbe28158ce820e3e1c56a"} {"id":"scuba:admin_004_001","source_id":"scuba","original_id":"admin_004_001","kind":"benchmark_task","instruction":"Could you spin up a summary report on Opportunities named SalesByStageRPT, grouped by the Stage field? Then add a Bar chart titled \"Opportunity Counts by Stage\" and turn on Show Values.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_001","query_templated":"Create a summary report with the Opportunities type with name SalesByStageRPT on the Stage field. Then create a Bar chart with chart title Opportunity Counts by Stage and check the box for [Show Values].","query":"Could you spin up a summary report on Opportunities named SalesByStageRPT, grouped by the Stage field? Then add a Bar chart titled \"Opportunity Counts by Stage\" and turn on Show Values.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Opportunities","report_name":"SalesByStageRPT","grouping_field":"STAGE_NAME","chart_type":"Horizontal Bar","chart_title":"Opportunity Counts by Stage"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"SalesByStageRPT","grouping_field":"Stage","chart_type":"Bar","chart_title":"Opportunity Counts by Stage","additional_comments":"and check the box for [Show Values]."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/15","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_004","template_name":"create_summary_report_with_chart","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/15"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/15"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_summary_report_with_chart"}],"source_schema_notes":[],"template_registry":{"name":"create_summary_report_with_chart","original":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nCould you spin up a summary report on Opportunities named SalesByStageRPT, grouped by the Stage field? Then add a Bar chart titled \"Opportunity Counts by Stage\" and turn on Show Values.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_001","query_templated":"Create a summary report with the Opportunities type with name SalesByStageRPT on the Stage field. Then create a Bar chart with chart title Opportunity Counts by Stage and check the box for [Show Values].","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nCould you spin up a summary report on Opportunities named SalesByStageRPT, grouped by the Stage field? Then add a Bar chart titled \"Opportunity Counts by Stage\" and turn on Show Values.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Opportunities","report_name":"SalesByStageRPT","grouping_field":"STAGE_NAME","chart_type":"Horizontal Bar","chart_title":"Opportunity Counts by Stage"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"SalesByStageRPT","grouping_field":"Stage","chart_type":"Bar","chart_title":"Opportunity Counts by Stage","additional_comments":"and check the box for [Show Values]."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/15"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L509","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_summary_report_with_chart","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"dbf50157085c47e53aa179d3cca23c0ca2226304b3cc137b6cf01f72fefbfeb8"} {"id":"scuba:admin_004_002","source_id":"scuba","original_id":"admin_004_002","kind":"benchmark_task","instruction":"In Reports, build a summary report for Accounts and save it as MyAccountRPT-1, grouped by Rating. Attach a Donut chart titled \"record counts by rating\" with the legend positioned on the bottom.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_002","query_templated":"Create a summary report with the Accounts type with name MyAccountRPT-1 on the Rating field. Then create a Donut chart with chart title record counts by rating and put the legend on [bottom].","query":"In Reports, build a summary report for Accounts and save it as MyAccountRPT-1, grouped by Rating. Attach a Donut chart titled \"record counts by rating\" with the legend positioned on the bottom.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountRPT-1","grouping_field":"RATING","chart_type":"Donut","chart_title":"record counts by rating"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountRPT-1","grouping_field":"Rating","chart_type":"Donut","chart_title":"record counts by rating","additional_comments":"and put the legend on [bottom]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/16","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_004","template_name":"create_summary_report_with_chart","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/16"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/16"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_summary_report_with_chart"}],"source_schema_notes":[],"template_registry":{"name":"create_summary_report_with_chart","original":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nIn Reports, build a summary report for Accounts and save it as MyAccountRPT-1, grouped by Rating. Attach a Donut chart titled \"record counts by rating\" with the legend positioned on the bottom.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_002","query_templated":"Create a summary report with the Accounts type with name MyAccountRPT-1 on the Rating field. Then create a Donut chart with chart title record counts by rating and put the legend on [bottom].","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nIn Reports, build a summary report for Accounts and save it as MyAccountRPT-1, grouped by Rating. Attach a Donut chart titled \"record counts by rating\" with the legend positioned on the bottom.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Accounts","report_name":"MyAccountRPT-1","grouping_field":"RATING","chart_type":"Donut","chart_title":"record counts by rating"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"MyAccountRPT-1","grouping_field":"Rating","chart_type":"Donut","chart_title":"record counts by rating","additional_comments":"and put the legend on [bottom]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/16"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L509","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_summary_report_with_chart","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0e8f9e4b09f9cd6ae7141383bb6b79dde32ea08cf433a30ed2a01b15158fddc2"} {"id":"scuba:admin_004_003","source_id":"scuba","original_id":"admin_004_003","kind":"benchmark_task","instruction":"Step 1: Create a Leads summary report called LeadSourceSummaryRPT and group it by the Lead Source field. Step 2: Insert a Funnel chart titled \"Leads by Source,\" set the legend on the bottom, and enable both Show Values and Show Percentages.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_003","query_templated":"Create a summary report with the Leads type with name LeadSourceSummaryRPT on the Lead Source field. Then create a Funnel chart with chart title Leads by Source put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].","query":"Step 1: Create a Leads summary report called LeadSourceSummaryRPT and group it by the Lead Source field. Step 2: Insert a Funnel chart titled \"Leads by Source,\" set the legend on the bottom, and enable both Show Values and Show Percentages.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Leads","report_name":"LeadSourceSummaryRPT","grouping_field":"LEAD_SOURCE","chart_type":"Funnel","chart_title":"Leads by Source"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"LeadSourceSummaryRPT","grouping_field":"Lead Source","chart_type":"Funnel","chart_title":"Leads by Source","additional_comments":"put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages]."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/17","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_004","template_name":"create_summary_report_with_chart","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/17"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/17"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_summary_report_with_chart"}],"source_schema_notes":[],"template_registry":{"name":"create_summary_report_with_chart","original":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nStep 1: Create a Leads summary report called LeadSourceSummaryRPT and group it by the Lead Source field. Step 2: Insert a Funnel chart titled \"Leads by Source,\" set the legend on the bottom, and enable both Show Values and Show Percentages.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_003","query_templated":"Create a summary report with the Leads type with name LeadSourceSummaryRPT on the Lead Source field. Then create a Funnel chart with chart title Leads by Source put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nStep 1: Create a Leads summary report called LeadSourceSummaryRPT and group it by the Lead Source field. Step 2: Insert a Funnel chart titled \"Leads by Source,\" set the legend on the bottom, and enable both Show Values and Show Percentages.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Leads","report_name":"LeadSourceSummaryRPT","grouping_field":"LEAD_SOURCE","chart_type":"Funnel","chart_title":"Leads by Source"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Leads","report_name":"LeadSourceSummaryRPT","grouping_field":"Lead Source","chart_type":"Funnel","chart_title":"Leads by Source","additional_comments":"put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages]."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/17"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L509","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_summary_report_with_chart","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8a0b4893c623e5f7901c0bd1c105a14efdece497aae1bc28c2e124896a3bfd61"} {"id":"scuba:admin_004_004","source_id":"scuba","original_id":"admin_004_004","kind":"benchmark_task","instruction":"Please create a summary report for Cases named CasePrioritySummaryRPT, grouped on Priority. After that, add a Column chart titled \"Cases by Priority\" and make sure Show Values is enabled.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_004","query_templated":"Create a summary report with the Cases type with name CasePrioritySummaryRPT on the Priority field. Then create a Column chart with chart title Cases by Priority and check the box for [Show Values].","query":"Please create a summary report for Cases named CasePrioritySummaryRPT, grouped on Priority. After that, add a Column chart titled \"Cases by Priority\" and make sure Show Values is enabled.","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Cases","report_name":"CasePrioritySummaryRPT","grouping_field":"PRIORITY","chart_type":"Vertical Bar","chart_title":"Cases by Priority"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CasePrioritySummaryRPT","grouping_field":"Priority","chart_type":"Column","chart_title":"Cases by Priority","additional_comments":"and check the box for [Show Values]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/18","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_004","template_name":"create_summary_report_with_chart","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/18"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/18"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_summary_report_with_chart"}],"source_schema_notes":[],"template_registry":{"name":"create_summary_report_with_chart","original":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nPlease create a summary report for Cases named CasePrioritySummaryRPT, grouped on Priority. After that, add a Column chart titled \"Cases by Priority\" and make sure Show Values is enabled.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_004","query_templated":"Create a summary report with the Cases type with name CasePrioritySummaryRPT on the Priority field. Then create a Column chart with chart title Cases by Priority and check the box for [Show Values].","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nPlease create a summary report for Cases named CasePrioritySummaryRPT, grouped on Priority. After that, add a Column chart titled \"Cases by Priority\" and make sure Show Values is enabled.\n```","human_review_comment":"Accept","ground_truth_dict":{"object_name":"Cases","report_name":"CasePrioritySummaryRPT","grouping_field":"PRIORITY","chart_type":"Vertical Bar","chart_title":"Cases by Priority"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CasePrioritySummaryRPT","grouping_field":"Priority","chart_type":"Column","chart_title":"Cases by Priority","additional_comments":"and check the box for [Show Values]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/18"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L509","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_summary_report_with_chart","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"da6acd5cd5ad77a3cf86c3f53a597607b4ba9f218691de8ac6ef87bf2dbb3a8b"} {"id":"scuba:admin_004_005","source_id":"scuba","original_id":"admin_004_005","kind":"benchmark_task","instruction":"Set up a Contacts summary report named ContactSourceRPT on the Source field. Then add a Scatter Plot chart with the title \"Contacts by Source,\" and place the legend on the bottom.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_005","query_templated":"Create a summary report with the Contacts type with name ContactSourceRPT on the Source field. Then create a Scatter Plot chart with chart title Contacts by Source and put the legend on [bottom].","query":"Set up a Contacts summary report named ContactSourceRPT on the Source field. Then add a Scatter Plot chart with the title \"Contacts by Source,\" and place the legend on the bottom.","human_review_comment":"change source field","ground_truth_dict":{"object_name":"Contacts & Accounts","report_name":"ContactSourceRPT","grouping_field":"CONTACT_SOURCE","chart_type":"Scatter","chart_title":"Contacts by Source"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Contacts","report_name":"ContactSourceRPT","grouping_field":"Source","chart_type":"Scatter Plot","chart_title":"Contacts by Source","additional_comments":"and put the legend on [bottom]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/19","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_004","template_name":"create_summary_report_with_chart","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/19"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/19"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_summary_report_with_chart"}],"source_schema_notes":[],"template_registry":{"name":"create_summary_report_with_chart","original":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_summary_report_with_chart","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nSet up a Contacts summary report named ContactSourceRPT on the Source field. Then add a Scatter Plot chart with the title \"Contacts by Source,\" and place the legend on the bottom.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_004_005","query_templated":"Create a summary report with the Contacts type with name ContactSourceRPT on the Source field. Then create a Scatter Plot chart with chart title Contacts by Source and put the legend on [bottom].","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save.\n\n## Your ultimate task\n```\nSet up a Contacts summary report named ContactSourceRPT on the Source field. Then add a Scatter Plot chart with the title \"Contacts by Source,\" and place the legend on the bottom.\n```","human_review_comment":"change source field","ground_truth_dict":{"object_name":"Contacts & Accounts","report_name":"ContactSourceRPT","grouping_field":"CONTACT_SOURCE","chart_type":"Scatter","chart_title":"Contacts by Source"},"difficulty":"medium","query_template_name":"create_summary_report_with_chart","query_template_metadata":{"template_id":"admin_004","template_string":"Create a summary report with the {object_name} type with name {report_name} on the {grouping_field} field. Then create a {chart_type} chart with chart title {chart_title} {additional_comments}","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Contacts","report_name":"ContactSourceRPT","grouping_field":"Source","chart_type":"Scatter Plot","chart_title":"Contacts by Source","additional_comments":"and put the legend on [bottom]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a summary report with the [Opportunities] type named [SalesByStageRPT] on the [Stage] field. Then create a [Bar chart] with a chart title [Opportunity Counts by Stage] and check the box for [Show Values].\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce.\n- Open App Launcher, search for and open Reports.\n- Click New Report, select Opportunities, and Start Report.\n- In Outline, add a group on the Stage field (group rows by Stage).\n- Add a chart, set the chart type to Bar chart.\n- Open Chart Properties:\n - Set Chart Title to \"Opportunity Counts by Stage\".\n - Check \"Show Values\".\n- Save the report and name it \"SalesByStageRPT\".\nSample 2:\nTASK:Create a summary report with the [Leads] type named [LeadSourceSummaryRPT] on the [Lead Source] field. Then create a [Funnel chart] with chart title [Leads by Source], put the legend on [bottom], and check the boxes for [Show Values] and [Show Percentages].\nEXPERIENCE:\nSuccessful Plan Summary\n\n- Open Reports:\n - Log in, open App Launcher, search for “Reports,” and open the Reports app.\n - Click New Report, choose the Leads report type, and Start Report.\n\n- Group by Lead Source:\n - Use Add Group and select Lead Source to group the report.\n\n- Set filters:\n - Show Me: All Leads.\n - Created Date range: All Time.\n - Apply the filters.\n\n- Add and configure chart:\n - Add Chart and open Chart Properties.\n - Set chart type to Funnel.\n - Chart Title: Leads by Source.\n - Enable Show Values and Show Percentages.\n - Set Legend position to Bottom.\n\n- Save the report:\n - Click Save & Run.\n - Name the report LeadSourceSummaryRPT and Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/19"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L509","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_summary_report_with_chart","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"afc9ac234e8bf4fae3d14ec37ffe31b41640e72bc4081d27c960a4a6bd2bf276"} {"id":"scuba:admin_005_001","source_id":"scuba","original_id":"admin_005_001","kind":"benchmark_task","instruction":"Please create a matrix report based on Opportunities and name it “OpptyMatrixByMonthType.” Use a SUM aggregation on the Amount field. Group the rows by Close Month and the columns by Type. Save it in the Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_001","query_templated":"Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.","query":"Please create a matrix report based on Opportunities and name it “OpptyMatrixByMonthType.” Use a SUM aggregation on the Amount field. Group the rows by Close Month and the columns by Type. Save it in the Public Reports.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunities","report_name":"OpptyMatrixByMonthType","field_name":"s!AMOUNT","operation":"Sum","row_group_field":"CLOSE_MONTH","column_group_field":"TYPE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"OpptyMatrixByMonthType","field_name":"Amount","operation":"SUM","row_group_field":"Close Month","column_group_field":"Type"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/20","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_005","template_name":"create_matrix_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/20"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/20"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matrix_report"}],"source_schema_notes":[],"template_registry":{"name":"create_matrix_report","original":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nPlease create a matrix report based on Opportunities and name it “OpptyMatrixByMonthType.” Use a SUM aggregation on the Amount field. Group the rows by Close Month and the columns by Type. Save it in the Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_001","query_templated":"Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nPlease create a matrix report based on Opportunities and name it “OpptyMatrixByMonthType.” Use a SUM aggregation on the Amount field. Group the rows by Close Month and the columns by Type. Save it in the Public Reports.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunities","report_name":"OpptyMatrixByMonthType","field_name":"s!AMOUNT","operation":"Sum","row_group_field":"CLOSE_MONTH","column_group_field":"TYPE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Opportunities","report_name":"OpptyMatrixByMonthType","field_name":"Amount","operation":"SUM","row_group_field":"Close Month","column_group_field":"Type"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/20"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L547","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matrix_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"15e9b9a4ea04174e00b1cd30b5dd4358f88c48f4afa5d50801a250fcbac10438"} {"id":"scuba:admin_005_002","source_id":"scuba","original_id":"admin_005_002","kind":"benchmark_task","instruction":"Build me a matrix report from Accounts called “AccountMatrixOwnerSite.” I want the Annual Revenue field summarized using Average. Group rows by Account Owner and columns by Account Site, then store it under Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_002","query_templated":"Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.","query":"Build me a matrix report from Accounts called “AccountMatrixOwnerSite.” I want the Annual Revenue field summarized using Average. Group rows by Account Owner and columns by Account Site, then store it under Public Reports.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Accounts","report_name":"AccountMatrixOwnerSite","field_name":"a!SALES","operation":"Average","row_group_field":"USERS.NAME","column_group_field":"SITE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"AccountMatrixOwnerSite","field_name":"Annual Revenue","operation":"Average","row_group_field":"Account Owner","column_group_field":"Account Site"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/21","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_005","template_name":"create_matrix_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/21"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/21"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matrix_report"}],"source_schema_notes":[],"template_registry":{"name":"create_matrix_report","original":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nBuild me a matrix report from Accounts called “AccountMatrixOwnerSite.” I want the Annual Revenue field summarized using Average. Group rows by Account Owner and columns by Account Site, then store it under Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_002","query_templated":"Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nBuild me a matrix report from Accounts called “AccountMatrixOwnerSite.” I want the Annual Revenue field summarized using Average. Group rows by Account Owner and columns by Account Site, then store it under Public Reports.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Accounts","report_name":"AccountMatrixOwnerSite","field_name":"a!SALES","operation":"Average","row_group_field":"USERS.NAME","column_group_field":"SITE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Accounts","report_name":"AccountMatrixOwnerSite","field_name":"Annual Revenue","operation":"Average","row_group_field":"Account Owner","column_group_field":"Account Site"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/21"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L547","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matrix_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0c44c5a7084c29f60f321024fdd926911a969956dfde721d7ced60153891b4e8"} {"id":"scuba:admin_005_003","source_id":"scuba","original_id":"admin_005_003","kind":"benchmark_task","instruction":"I need a matrix report on Cases titled “CaseMatrixSubjectPriority.” Roll up the Age field using the Maximum value. Set the row grouping to Subject and the column grouping to Priority, and save the report to Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_003","query_templated":"Build a matrix report of the Cases type with name CaseMatrixSubjectPriority. Summarize on the Age column with the Maximum operation. Group row on the Subject field and group column on the Priority field. Save it to the public report.","query":"I need a matrix report on Cases titled “CaseMatrixSubjectPriority.” Roll up the Age field using the Maximum value. Set the row grouping to Subject and the column grouping to Priority, and save the report to Public Reports.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Cases","report_name":"CaseMatrixSubjectPriority","field_name":"mx!AGE","operation":"Largest","row_group_field":"SUBJECT","column_group_field":"PRIORITY"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CaseMatrixSubjectPriority","field_name":"Age","operation":"Maximum","row_group_field":"Subject","column_group_field":"Priority"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/22","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_005","template_name":"create_matrix_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/22"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/22"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matrix_report"}],"source_schema_notes":[],"template_registry":{"name":"create_matrix_report","original":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nI need a matrix report on Cases titled “CaseMatrixSubjectPriority.” Roll up the Age field using the Maximum value. Set the row grouping to Subject and the column grouping to Priority, and save the report to Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_003","query_templated":"Build a matrix report of the Cases type with name CaseMatrixSubjectPriority. Summarize on the Age column with the Maximum operation. Group row on the Subject field and group column on the Priority field. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nI need a matrix report on Cases titled “CaseMatrixSubjectPriority.” Roll up the Age field using the Maximum value. Set the row grouping to Subject and the column grouping to Priority, and save the report to Public Reports.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Cases","report_name":"CaseMatrixSubjectPriority","field_name":"mx!AGE","operation":"Largest","row_group_field":"SUBJECT","column_group_field":"PRIORITY"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Cases","report_name":"CaseMatrixSubjectPriority","field_name":"Age","operation":"Maximum","row_group_field":"Subject","column_group_field":"Priority"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/22"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L547","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matrix_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"406219c8628e1ecbd074d7cf541f247d0fcf63c95d821ec62126d259d417fbaf"} {"id":"scuba:admin_005_004","source_id":"scuba","original_id":"admin_005_004","kind":"benchmark_task","instruction":"Set up a matrix report for Products with the name “ProductMatrixFamilyDate.” Aggregate the Active field using Sum. Organize rows by Product Family and columns by Created Date. Save this to the Public Reports folder.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_004","query_templated":"Build a matrix report of the Products type with name ProductMatrixFamilyDate. Summarize on the Active column with the Sum operation. Group row on the Product Family field and group column on the Created Date field. Save it to the public report.","query":"Set up a matrix report for Products with the name “ProductMatrixFamilyDate.” Aggregate the Active field using Sum. Organize rows by Product Family and columns by Created Date. Save this to the Public Reports folder.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Products","report_name":"ProductMatrixFamilyDate","field_name":"s!ACTIVE","operation":"Sum","row_group_field":"FAMILY","column_group_field":"CREATED_DATE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Products","report_name":"ProductMatrixFamilyDate","field_name":"Active","operation":"Sum","row_group_field":"Product Family","column_group_field":"Created Date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/23","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_005","template_name":"create_matrix_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/23"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/23"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matrix_report"}],"source_schema_notes":[],"template_registry":{"name":"create_matrix_report","original":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nSet up a matrix report for Products with the name “ProductMatrixFamilyDate.” Aggregate the Active field using Sum. Organize rows by Product Family and columns by Created Date. Save this to the Public Reports folder.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_004","query_templated":"Build a matrix report of the Products type with name ProductMatrixFamilyDate. Summarize on the Active column with the Sum operation. Group row on the Product Family field and group column on the Created Date field. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nSet up a matrix report for Products with the name “ProductMatrixFamilyDate.” Aggregate the Active field using Sum. Organize rows by Product Family and columns by Created Date. Save this to the Public Reports folder.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Products","report_name":"ProductMatrixFamilyDate","field_name":"s!ACTIVE","operation":"Sum","row_group_field":"FAMILY","column_group_field":"CREATED_DATE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Products","report_name":"ProductMatrixFamilyDate","field_name":"Active","operation":"Sum","row_group_field":"Product Family","column_group_field":"Created Date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/23"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L547","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matrix_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3423d69e0c854bb2a563002c4a25c6a238ebe1e6024ce35fdfee348a864f948e"} {"id":"scuba:admin_005_005","source_id":"scuba","original_id":"admin_005_005","kind":"benchmark_task","instruction":"Create a matrix report for Campaigns named “CampaignMatrixStatusType.” Summarize Budgeted Cost with the Min operation. Group rows by Status and columns by Type, and place it in Public Reports.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_005","query_templated":"Build a matrix report of the Campaigns type with name CampaignMatrixStatusType. Summarize on the Budgeted Cost column with the Min operation. Group row on the Status field and group column on the Type field. Save it to the public report.","query":"Create a matrix report for Campaigns named “CampaignMatrixStatusType.” Summarize Budgeted Cost with the Min operation. Group rows by Status and columns by Type, and place it in Public Reports.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Campaigns","report_name":"CampaignMatrixStatusType","field_name":"m!BUDGETED_COST","operation":"Smallest","row_group_field":"CAMPAIGN_STATUS","column_group_field":"CAMPAIGN_TYPE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Campaigns","report_name":"CampaignMatrixStatusType","field_name":"Budgeted Cost","operation":"Min","row_group_field":"Status","column_group_field":"Type"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/24","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_005","template_name":"create_matrix_report","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/24"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/24"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matrix_report"}],"source_schema_notes":[],"template_registry":{"name":"create_matrix_report","original":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matrix_report","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nCreate a matrix report for Campaigns named “CampaignMatrixStatusType.” Summarize Budgeted Cost with the Min operation. Group rows by Status and columns by Type, and place it in Public Reports.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_005_005","query_templated":"Build a matrix report of the Campaigns type with name CampaignMatrixStatusType. Summarize on the Budgeted Cost column with the Min operation. Group row on the Status field and group column on the Type field. Save it to the public report.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save.\n\n## Your ultimate task\n```\nCreate a matrix report for Campaigns named “CampaignMatrixStatusType.” Summarize Budgeted Cost with the Min operation. Group rows by Status and columns by Type, and place it in Public Reports.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Campaigns","report_name":"CampaignMatrixStatusType","field_name":"m!BUDGETED_COST","operation":"Smallest","row_group_field":"CAMPAIGN_STATUS","column_group_field":"CAMPAIGN_TYPE"},"difficulty":"medium","query_template_name":"create_matrix_report","query_template_metadata":{"template_id":"admin_005","template_string":"Build a matrix report of the {object_name} type with name {report_name}. Summarize on the {field_name} column with the {operation} operation. Group row on the {row_group_field} field and group column on the {column_group_field} field. Save it to the public report.","prerequisites":{},"objects":[],"metadata_types":["Report"]},"instance_dict":{"object_name":"Campaigns","report_name":"CampaignMatrixStatusType","field_name":"Budgeted Cost","operation":"Min","row_group_field":"Status","column_group_field":"Type"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build a matrix report of the Opportunities type with name OpptyMatrixByMonthType. Summarize on the Amount column with the SUM operation. Group row on the Close Month field and group column on the Type field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report (Salesforce Lightning):\n\n- Navigate to Reports:\n - Open App Launcher, search for “Reports”, and open it.\n - Click New Report, choose report type “Opportunities”, then Start Report.\n\n- Configure as a matrix:\n - Group Rows by Close Date, then set Group Date By to Calendar Month (Close Month).\n - Group Columns by Type.\n - Summarize the Amount column using Sum.\n\n- Save and run:\n - Click Save & Run.\n - Name: OpptyMatrixByMonthType.\n - Select folder: Public Reports.\n - Save.\n\nNotes:\n- No hotkeys were used.\n- Adding both a row and a column grouping automatically creates a matrix report.\nSample 2:\nTASK:Build a matrix report of the Accounts type with name AccountMatrixOwnerSite. Summarize on the Annual Revenue column with the Average operation. Group row on the Account Owner field and group column on the Account Site field. Save it to the public report.\nEXPERIENCE:\nSuccessful plan to build the matrix report:\n\n- Log in to Salesforce and open the App Launcher.\n- Search for and open Reports.\n- Click New Report, select Accounts as the report type, and click Start Report.\n- Create a matrix layout:\n - Group rows by Account Owner (use Group Rows search to add “Account Owner”).\n - Group columns by Account Site (use Group Columns search to add “Account Site”).\n- Add and summarize Annual Revenue:\n - Use the column search to add “Annual Revenue” if not already present.\n - Open Annual Revenue column actions > Summarize > choose Average.\n- Save the report:\n - Click Save & Run.\n - Name: AccountMatrixOwnerSite.\n - Choose folder: Public Reports.\n - Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/24"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L547","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matrix_report","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"98d66d0528bc1d2f128eeff12ac67ec2483ed262a57b454d989e300ade315851"} {"id":"scuba:admin_006_001","source_id":"scuba","original_id":"admin_006_001","kind":"benchmark_task","instruction":"Could you spin up a dashboard called Big Deals? Drop in a chart widget that uses the Required_Opportunities_Report report. Make the type a Table and title the chart Opportunities in Stages. If Required_Opportunities_Report isn’t available, create it first: report type Opportunities, name Required_Opportunities_Report, grouped by the Stage field.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_001","query_templated":"Create a dashboard with name Big Deals. Add a chart widget using the Required_Opportunities_Report report. The chart type should be Table and set the chart title to Opportunities in Stages. Note that if the Required_Opportunities_Report report does not exist, create it first by following the instructions below. Create a report with the Opportunities type with the name Required_Opportunities_Report. Group by Stage field.","query":"Could you spin up a dashboard called Big Deals? Drop in a chart widget that uses the Required_Opportunities_Report report. Make the type a Table and title the chart Opportunities in Stages. If Required_Opportunities_Report isn’t available, create it first: report type Opportunities, name Required_Opportunities_Report, grouped by the Stage field.","human_review_comment":"accept","ground_truth_dict":{"dashboard_name":"Big Deals","object_name":"Opportunities","report_name":"Required_Opportunities_Report","group_field":"STAGE_NAME","chart_type":"FlexTable","chart_title":"Opportunities in Stages"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Big Deals","object_name":"Opportunities","report_name":"Required_Opportunities_Report","group_field":"Stage","chart_type":"Table","chart_title":"Opportunities in Stages"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/25","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_006","template_name":"create_dashboard","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/25"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/25"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dashboard"}],"source_schema_notes":[],"template_registry":{"name":"create_dashboard","original":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you spin up a dashboard called Big Deals? Drop in a chart widget that uses the Required_Opportunities_Report report. Make the type a Table and title the chart Opportunities in Stages. If Required_Opportunities_Report isn’t available, create it first: report type Opportunities, name Required_Opportunities_Report, grouped by the Stage field.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_001","query_templated":"Create a dashboard with name Big Deals. Add a chart widget using the Required_Opportunities_Report report. The chart type should be Table and set the chart title to Opportunities in Stages. Note that if the Required_Opportunities_Report report does not exist, create it first by following the instructions below. Create a report with the Opportunities type with the name Required_Opportunities_Report. Group by Stage field.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you spin up a dashboard called Big Deals? Drop in a chart widget that uses the Required_Opportunities_Report report. Make the type a Table and title the chart Opportunities in Stages. If Required_Opportunities_Report isn’t available, create it first: report type Opportunities, name Required_Opportunities_Report, grouped by the Stage field.\n```","human_review_comment":"accept","ground_truth_dict":{"dashboard_name":"Big Deals","object_name":"Opportunities","report_name":"Required_Opportunities_Report","group_field":"STAGE_NAME","chart_type":"FlexTable","chart_title":"Opportunities in Stages"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Big Deals","object_name":"Opportunities","report_name":"Required_Opportunities_Report","group_field":"Stage","chart_type":"Table","chart_title":"Opportunities in Stages"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/25"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L591","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dashboard","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}},{"type":"soql","id":"dashboard_info","params":{"query":"SELECT Id FROM Dashboard WHERE Title='{dashboard_name}'"}},{"type":"tooling","id":"dashboard_metadata","params":{"endpoint":"/services/data/v62.0/analytics/dashboards/{retrieved_results['dashboard_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f233bd9e0b01798cb571a4c18017ba2ddc0e7fdab85d2aa73960ee9e43cab9cd"} {"id":"scuba:admin_006_002","source_id":"scuba","original_id":"admin_006_002","kind":"benchmark_task","instruction":"Please create a dashboard named Sales Performance. Add a chart widget sourced from the Required_Leads_by_Source report, set the chart type to Donut, and name it Leads by Source. If the Required_Leads_by_Source report isn’t already there, build it first as a Leads report named Required_Leads_by_Source, grouped by Lead Source.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_002","query_templated":"Create a dashboard with name Sales Performance. Add a chart widget using the Required_Leads_by_Source report. The chart type should be Donut and set the chart title to Leads by Source. Note that if the Required_Leads_by_Source report does not exist, create it first by following the instructions below. Create a report with the Leads type with the name Required_Leads_by_Source. Group by Lead Source field.","query":"Please create a dashboard named Sales Performance. Add a chart widget sourced from the Required_Leads_by_Source report, set the chart type to Donut, and name it Leads by Source. If the Required_Leads_by_Source report isn’t already there, build it first as a Leads report named Required_Leads_by_Source, grouped by Lead Source.","human_review_comment":"Change label to name","ground_truth_dict":{"dashboard_name":"Sales Performance","object_name":"Leads","report_name":"Required_Leads_by_Source","group_field":"LEAD_SOURCE","chart_type":"Donut","chart_title":"Leads by Source"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Sales Performance","object_name":"Leads","report_name":"Required_Leads_by_Source","group_field":"Lead Source","chart_type":"Donut","chart_title":"Leads by Source"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/26","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_006","template_name":"create_dashboard","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/26"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/26"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dashboard"}],"source_schema_notes":[],"template_registry":{"name":"create_dashboard","original":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease create a dashboard named Sales Performance. Add a chart widget sourced from the Required_Leads_by_Source report, set the chart type to Donut, and name it Leads by Source. If the Required_Leads_by_Source report isn’t already there, build it first as a Leads report named Required_Leads_by_Source, grouped by Lead Source.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_002","query_templated":"Create a dashboard with name Sales Performance. Add a chart widget using the Required_Leads_by_Source report. The chart type should be Donut and set the chart title to Leads by Source. Note that if the Required_Leads_by_Source report does not exist, create it first by following the instructions below. Create a report with the Leads type with the name Required_Leads_by_Source. Group by Lead Source field.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease create a dashboard named Sales Performance. Add a chart widget sourced from the Required_Leads_by_Source report, set the chart type to Donut, and name it Leads by Source. If the Required_Leads_by_Source report isn’t already there, build it first as a Leads report named Required_Leads_by_Source, grouped by Lead Source.\n```","human_review_comment":"Change label to name","ground_truth_dict":{"dashboard_name":"Sales Performance","object_name":"Leads","report_name":"Required_Leads_by_Source","group_field":"LEAD_SOURCE","chart_type":"Donut","chart_title":"Leads by Source"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Sales Performance","object_name":"Leads","report_name":"Required_Leads_by_Source","group_field":"Lead Source","chart_type":"Donut","chart_title":"Leads by Source"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/26"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L591","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dashboard","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}},{"type":"soql","id":"dashboard_info","params":{"query":"SELECT Id FROM Dashboard WHERE Title='{dashboard_name}'"}},{"type":"tooling","id":"dashboard_metadata","params":{"endpoint":"/services/data/v62.0/analytics/dashboards/{retrieved_results['dashboard_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2c91fe0d1fd611e820701dd996a551cf2c013f298461e8b74cbbc99ff22a58cb"} {"id":"scuba:admin_006_003","source_id":"scuba","original_id":"admin_006_003","kind":"benchmark_task","instruction":"First, create a new dashboard and give it the name Case Overview.\r\nOn that dashboard, add a chart widget that uses the Required_Cases_by_Priority report.\r\nSet the chart type to Stacked Bar and update the title to Cases by Priority. If the Required_Cases_by_Priority report isn’t already there, you’ll need to make it first:\r\n\r\n1. Create a report with the Cases type.\r\n\r\n2. Name it Required_Cases_by_Priority.\r\n\r\n3. Group the report by the Priority field.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_003","query_templated":"Create a dashboard with name Case Overview. Add a chart widget using the Required_Cases_by_Priority report. The chart type should be Stacked Bar and set the chart title to Cases by Priority. Note that if the Required_Cases_by_Priority report does not exist, create it first by following the instructions below. Create a report with the Cases type with the name Required_Cases_by_Priority. Group by Priority field.","query":"First, create a new dashboard and give it the name Case Overview.\r\nOn that dashboard, add a chart widget that uses the Required_Cases_by_Priority report.\r\nSet the chart type to Stacked Bar and update the title to Cases by Priority. If the Required_Cases_by_Priority report isn’t already there, you’ll need to make it first:\r\n\r\n1. Create a report with the Cases type.\r\n\r\n2. Name it Required_Cases_by_Priority.\r\n\r\n3. Group the report by the Priority field.","human_review_comment":"rewrite due to semantic issue","ground_truth_dict":{"dashboard_name":"Case Overview","object_name":"Cases","report_name":"Required_Cases_by_Priority","group_field":"PRIORITY","chart_type":"Bar","chart_title":"Cases by Priority"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Case Overview","object_name":"Cases","report_name":"Required_Cases_by_Priority","group_field":"Priority","chart_type":"Stacked Bar","chart_title":"Cases by Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/27","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_006","template_name":"create_dashboard","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/27"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/27"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dashboard"}],"source_schema_notes":[],"template_registry":{"name":"create_dashboard","original":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFirst, create a new dashboard and give it the name Case Overview.\r\nOn that dashboard, add a chart widget that uses the Required_Cases_by_Priority report.\r\nSet the chart type to Stacked Bar and update the title to Cases by Priority. If the Required_Cases_by_Priority report isn’t already there, you’ll need to make it first:\r\n\r\n1. Create a report with the Cases type.\r\n\r\n2. Name it Required_Cases_by_Priority.\r\n\r\n3. Group the report by the Priority field.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_003","query_templated":"Create a dashboard with name Case Overview. Add a chart widget using the Required_Cases_by_Priority report. The chart type should be Stacked Bar and set the chart title to Cases by Priority. Note that if the Required_Cases_by_Priority report does not exist, create it first by following the instructions below. Create a report with the Cases type with the name Required_Cases_by_Priority. Group by Priority field.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFirst, create a new dashboard and give it the name Case Overview.\r\nOn that dashboard, add a chart widget that uses the Required_Cases_by_Priority report.\r\nSet the chart type to Stacked Bar and update the title to Cases by Priority. If the Required_Cases_by_Priority report isn’t already there, you’ll need to make it first:\r\n\r\n1. Create a report with the Cases type.\r\n\r\n2. Name it Required_Cases_by_Priority.\r\n\r\n3. Group the report by the Priority field.\n```","human_review_comment":"rewrite due to semantic issue","ground_truth_dict":{"dashboard_name":"Case Overview","object_name":"Cases","report_name":"Required_Cases_by_Priority","group_field":"PRIORITY","chart_type":"Bar","chart_title":"Cases by Priority"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Case Overview","object_name":"Cases","report_name":"Required_Cases_by_Priority","group_field":"Priority","chart_type":"Stacked Bar","chart_title":"Cases by Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/27"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L591","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dashboard","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}},{"type":"soql","id":"dashboard_info","params":{"query":"SELECT Id FROM Dashboard WHERE Title='{dashboard_name}'"}},{"type":"tooling","id":"dashboard_metadata","params":{"endpoint":"/services/data/v62.0/analytics/dashboards/{retrieved_results['dashboard_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"284cf2bd4e136ec500987da8d0e570e52633f58205a53d6530b2d66ad19045a7"} {"id":"scuba:admin_006_004","source_id":"scuba","original_id":"admin_006_004","kind":"benchmark_task","instruction":"Hey, could you please set up a new dashboard called Account Health?\r\nOn it, add a chart widget that pulls from the Required Active Accounts report.\r\nSwitch the chart type to Column, and give it the title Active Accounts by Industry. If the Required Active Accounts report isn’t there yet, no worries — just create it first:\r\n\r\nBuild a new report with the Accounts type.\r\n\r\nName it Required Active Accounts.\r\n\r\nGroup the data by the Industry field.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_004","query_templated":"Create a dashboard with name Account Health. Add a chart widget using the Required Active Accounts report. The chart type should be Column and set the chart title to Active Accounts by Industry. Note that if the Required Active Accounts report does not exist, create it first by following the instructions below. Create a report with the Accounts type with the name Required Active Accounts. Group by Industry field.","query":"Hey, could you please set up a new dashboard called Account Health?\r\nOn it, add a chart widget that pulls from the Required Active Accounts report.\r\nSwitch the chart type to Column, and give it the title Active Accounts by Industry. If the Required Active Accounts report isn’t there yet, no worries — just create it first:\r\n\r\nBuild a new report with the Accounts type.\r\n\r\nName it Required Active Accounts.\r\n\r\nGroup the data by the Industry field.","human_review_comment":"rewrite due to semantic issue","ground_truth_dict":{"dashboard_name":"Account Health","object_name":"Accounts","report_name":"Required Active Accounts","group_field":"INDUSTRY","chart_type":"Column","chart_title":"Active Accounts by Industry"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Account Health","object_name":"Accounts","report_name":"Required Active Accounts","group_field":"Industry","chart_type":"Column","chart_title":"Active Accounts by Industry"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/28","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_006","template_name":"create_dashboard","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/28"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/28"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dashboard"}],"source_schema_notes":[],"template_registry":{"name":"create_dashboard","original":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nHey, could you please set up a new dashboard called Account Health?\r\nOn it, add a chart widget that pulls from the Required Active Accounts report.\r\nSwitch the chart type to Column, and give it the title Active Accounts by Industry. If the Required Active Accounts report isn’t there yet, no worries — just create it first:\r\n\r\nBuild a new report with the Accounts type.\r\n\r\nName it Required Active Accounts.\r\n\r\nGroup the data by the Industry field.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_004","query_templated":"Create a dashboard with name Account Health. Add a chart widget using the Required Active Accounts report. The chart type should be Column and set the chart title to Active Accounts by Industry. Note that if the Required Active Accounts report does not exist, create it first by following the instructions below. Create a report with the Accounts type with the name Required Active Accounts. Group by Industry field.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nHey, could you please set up a new dashboard called Account Health?\r\nOn it, add a chart widget that pulls from the Required Active Accounts report.\r\nSwitch the chart type to Column, and give it the title Active Accounts by Industry. If the Required Active Accounts report isn’t there yet, no worries — just create it first:\r\n\r\nBuild a new report with the Accounts type.\r\n\r\nName it Required Active Accounts.\r\n\r\nGroup the data by the Industry field.\n```","human_review_comment":"rewrite due to semantic issue","ground_truth_dict":{"dashboard_name":"Account Health","object_name":"Accounts","report_name":"Required Active Accounts","group_field":"INDUSTRY","chart_type":"Column","chart_title":"Active Accounts by Industry"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Account Health","object_name":"Accounts","report_name":"Required Active Accounts","group_field":"Industry","chart_type":"Column","chart_title":"Active Accounts by Industry"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/28"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L591","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dashboard","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}},{"type":"soql","id":"dashboard_info","params":{"query":"SELECT Id FROM Dashboard WHERE Title='{dashboard_name}'"}},{"type":"tooling","id":"dashboard_metadata","params":{"endpoint":"/services/data/v62.0/analytics/dashboards/{retrieved_results['dashboard_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d5085c2479a2e21a565320917f9fbc47850abce61996c420e39531824f068d35"} {"id":"scuba:admin_006_005","source_id":"scuba","original_id":"admin_006_005","kind":"benchmark_task","instruction":"Build a dashboard titled Active Products. Add a chart widget using the Required Products report, set the title to Active Products, and connect it to the Required Products report. If Required Products isn’t present, create it beforehand: report type Product2, name Required Products, grouped by the Active field.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_005","query_templated":"Create a dashboard with name Active Products. Add a chart widget using the Required Products report. The chart type should be Line and set the chart title to Active Products. Note that if the Required Products report does not exist, create it first by following the instructions below. Create a report with the Product2 type with the name Required Products. Group by Active field.","query":"Build a dashboard titled Active Products. Add a chart widget using the Required Products report, set the title to Active Products, and connect it to the Required Products report. If Required Products isn’t present, create it beforehand: report type Product2, name Required Products, grouped by the Active field.","human_review_comment":"Change the semantic order","ground_truth_dict":{"dashboard_name":"Active Products","object_name":"Product2","report_name":"Required Products","group_field":"ACTIVE","chart_type":"Line","chart_title":"Active Products"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Active Products","object_name":"Product2","report_name":"Required Products","group_field":"Active","chart_type":"Line","chart_title":"Active Products"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/29","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Report / Dashboard","source_difficulty":"medium","template_id":"admin_006","template_name":"create_dashboard","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/29"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/29"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dashboard"}],"source_schema_notes":[],"template_registry":{"name":"create_dashboard","original":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dashboard","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nBuild a dashboard titled Active Products. Add a chart widget using the Required Products report, set the title to Active Products, and connect it to the Required Products report. If Required Products isn’t present, create it beforehand: report type Product2, name Required Products, grouped by the Active field.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Report / Dashboard","task_id":"admin_006_005","query_templated":"Create a dashboard with name Active Products. Add a chart widget using the Required Products report. The chart type should be Line and set the chart title to Active Products. Note that if the Required Products report does not exist, create it first by following the instructions below. Create a report with the Product2 type with the name Required Products. Group by Active field.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nBuild a dashboard titled Active Products. Add a chart widget using the Required Products report, set the title to Active Products, and connect it to the Required Products report. If Required Products isn’t present, create it beforehand: report type Product2, name Required Products, grouped by the Active field.\n```","human_review_comment":"Change the semantic order","ground_truth_dict":{"dashboard_name":"Active Products","object_name":"Product2","report_name":"Required Products","group_field":"ACTIVE","chart_type":"Line","chart_title":"Active Products"},"difficulty":"medium","query_template_name":"create_dashboard","query_template_metadata":{"template_id":"admin_006","template_string":"Create a dashboard with name {dashboard_name}. Add a chart widget using the {report_name} report. The chart type should be {chart_type} and set the chart title to {chart_title}. Note that if the {report_name} report does not exist, create it first by following the instructions below. Create a report with the {object_name} type with the name {report_name}. Group by {group_field} field.","prerequisites":{"metadata":{"types_and_members":{"Report":["unfiled$public/Required_Active_Accounts_dV2","unfiled$public/Required_Cases_by_Priority_7OT","unfiled$public/Required_Leads_by_Source_hJ7","unfiled$public/Required_Opportunities_Report_TjH3","unfiled$public/Required_Products_dHv"]}}},"objects":[],"metadata_types":["Dashboard"]},"instance_dict":{"dashboard_name":"Active Products","object_name":"Product2","report_name":"Required Products","group_field":"Active","chart_type":"Line","chart_title":"Active Products"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/29"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L591","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dashboard","data_retrieval_spec":[{"type":"soql","id":"report_info","params":{"query":"SELECT Id FROM Report WHERE Name='{report_name}'"}},{"type":"tooling","id":"report_metadata","params":{"endpoint":"/services/data/v62.0/analytics/reports/{retrieved_results['report_info'].records[0]['Id']}/describe"}},{"type":"soql","id":"dashboard_info","params":{"query":"SELECT Id FROM Dashboard WHERE Title='{dashboard_name}'"}},{"type":"tooling","id":"dashboard_metadata","params":{"endpoint":"/services/data/v62.0/analytics/dashboards/{retrieved_results['dashboard_info'].records[0]['Id']}/describe"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"38f5156a37eae742b822a91606d128b8d1b682951ee48cd8838108b2a1d4b44e"} {"id":"scuba:admin_007_001","source_id":"scuba","original_id":"admin_007_001","kind":"benchmark_task","instruction":"On the Account object, set up a dynamic form and AnnualRevenue only shows up when the Active field is set to Yes; otherwise keep it hidden.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_001","query_templated":"Create a dynamic form on the Account object. The field AnnualRevenue should be visible only when Active's value equals Yes.","query":"On the Account object, set up a dynamic form and AnnualRevenue only shows up when the Active field is set to Yes; otherwise keep it hidden.","human_review_comment":"Change the logic","ground_truth_dict":{"object_name":"Account","field_name":"Record.AnnualRevenue","field_2":"{!Record.Active__c}","operator":"EQUAL","value":"Yes"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Account","field_name":"AnnualRevenue","field_2":"Active","operator":"equals","value":"Yes"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/30","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Dynamic Forms","source_difficulty":"medium","template_id":"admin_007","template_name":"create_dynamic_forms","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/30"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/30"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dynamic_forms"}],"source_schema_notes":[],"template_registry":{"name":"create_dynamic_forms","original":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nOn the Account object, set up a dynamic form and AnnualRevenue only shows up when the Active field is set to Yes; otherwise keep it hidden.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_001","query_templated":"Create a dynamic form on the Account object. The field AnnualRevenue should be visible only when Active's value equals Yes.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nOn the Account object, set up a dynamic form and AnnualRevenue only shows up when the Active field is set to Yes; otherwise keep it hidden.\n```","human_review_comment":"Change the logic","ground_truth_dict":{"object_name":"Account","field_name":"Record.AnnualRevenue","field_2":"{!Record.Active__c}","operator":"EQUAL","value":"Yes"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Account","field_name":"AnnualRevenue","field_2":"Active","operator":"equals","value":"Yes"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/30"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L660","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dynamic_forms","data_retrieval_spec":[{"type":"metadata","id":"flexipage_metadata","params":{"types_and_members":{"FlexiPage":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"1a68ed822570dd75bc83310661745d1d7e5cf706c853967ee6003debe8d03c5b"} {"id":"scuba:admin_007_002","source_id":"scuba","original_id":"admin_007_002","kind":"benchmark_task","instruction":"Could you please set up a dynamic form on the Contact object?\r\nMake sure the MobilePhone field only shows up when DoNotCall is set to false.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_002","query_templated":"Create a dynamic form on the Contact object. The field MobilePhone should be visible only when DoNotCall's value equals false.","query":"Could you please set up a dynamic form on the Contact object?\r\nMake sure the MobilePhone field only shows up when DoNotCall is set to false.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Contact","field_name":"Record.MobilePhone","field_2":"{!Record.DoNotCall}","operator":"EQUAL","value":"false"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Contact","field_name":"MobilePhone","field_2":"DoNotCall","operator":"equals","value":"false"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/31","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Dynamic Forms","source_difficulty":"medium","template_id":"admin_007","template_name":"create_dynamic_forms","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/31"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/31"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dynamic_forms"}],"source_schema_notes":[],"template_registry":{"name":"create_dynamic_forms","original":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you please set up a dynamic form on the Contact object?\r\nMake sure the MobilePhone field only shows up when DoNotCall is set to false.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_002","query_templated":"Create a dynamic form on the Contact object. The field MobilePhone should be visible only when DoNotCall's value equals false.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you please set up a dynamic form on the Contact object?\r\nMake sure the MobilePhone field only shows up when DoNotCall is set to false.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Contact","field_name":"Record.MobilePhone","field_2":"{!Record.DoNotCall}","operator":"EQUAL","value":"false"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Contact","field_name":"MobilePhone","field_2":"DoNotCall","operator":"equals","value":"false"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/31"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L660","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dynamic_forms","data_retrieval_spec":[{"type":"metadata","id":"flexipage_metadata","params":{"types_and_members":{"FlexiPage":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"1c3f30ada89ceda5baa9ef62c3b205851210c9b87298de83f5d68499b0f80929"} {"id":"scuba:admin_007_003","source_id":"scuba","original_id":"admin_007_003","kind":"benchmark_task","instruction":"Please set up a dynamic form on the Lead object.\r\nKeep the NumberOfEmployees field hidden by default, and only make it visible if its value is greater than 5,000.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_003","query_templated":"Create a dynamic form on the Lead object. The field NumberOfEmployees should be visible only when NumberOfEmployees's value greater_than 5000.","query":"Please set up a dynamic form on the Lead object.\r\nKeep the NumberOfEmployees field hidden by default, and only make it visible if its value is greater than 5,000.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Lead","field_name":"Record.NumberOfEmployees","field_2":"{!Record.NumberOfEmployees}","operator":"GT","value":"5000"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Lead","field_name":"NumberOfEmployees","field_2":"NumberOfEmployees","operator":"greater_than","value":5000},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/32","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Dynamic Forms","source_difficulty":"medium","template_id":"admin_007","template_name":"create_dynamic_forms","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/32"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/32"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dynamic_forms"}],"source_schema_notes":[],"template_registry":{"name":"create_dynamic_forms","original":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease set up a dynamic form on the Lead object.\r\nKeep the NumberOfEmployees field hidden by default, and only make it visible if its value is greater than 5,000.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_003","query_templated":"Create a dynamic form on the Lead object. The field NumberOfEmployees should be visible only when NumberOfEmployees's value greater_than 5000.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease set up a dynamic form on the Lead object.\r\nKeep the NumberOfEmployees field hidden by default, and only make it visible if its value is greater than 5,000.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Lead","field_name":"Record.NumberOfEmployees","field_2":"{!Record.NumberOfEmployees}","operator":"GT","value":"5000"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Lead","field_name":"NumberOfEmployees","field_2":"NumberOfEmployees","operator":"greater_than","value":5000},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/32"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L660","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dynamic_forms","data_retrieval_spec":[{"type":"metadata","id":"flexipage_metadata","params":{"types_and_members":{"FlexiPage":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"013626fc4da6b9ce0411d59ac034531a4e8bf527cf830a451c1144efcce461cf"} {"id":"scuba:admin_007_004","source_id":"scuba","original_id":"admin_007_004","kind":"benchmark_task","instruction":"Configure the Opportunity dynamic form so ExpectedRevenue appears only when Amount is less than 500; hide it for 500 and above.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_004","query_templated":"Create a dynamic form on the Opportunity object. The field ExpectedRevenue should be visible only when Amount's value less than 500.","query":"Configure the Opportunity dynamic form so ExpectedRevenue appears only when Amount is less than 500; hide it for 500 and above.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","field_name":"Record.ExpectedRevenue","field_2":"{!Record.Amount}","operator":"LT","value":"500"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Opportunity","field_name":"ExpectedRevenue","field_2":"Amount","operator":"less than","value":500},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/33","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Dynamic Forms","source_difficulty":"medium","template_id":"admin_007","template_name":"create_dynamic_forms","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/33"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/33"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dynamic_forms"}],"source_schema_notes":[],"template_registry":{"name":"create_dynamic_forms","original":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nConfigure the Opportunity dynamic form so ExpectedRevenue appears only when Amount is less than 500; hide it for 500 and above.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_004","query_templated":"Create a dynamic form on the Opportunity object. The field ExpectedRevenue should be visible only when Amount's value less than 500.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nConfigure the Opportunity dynamic form so ExpectedRevenue appears only when Amount is less than 500; hide it for 500 and above.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","field_name":"Record.ExpectedRevenue","field_2":"{!Record.Amount}","operator":"LT","value":"500"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Opportunity","field_name":"ExpectedRevenue","field_2":"Amount","operator":"less than","value":500},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/33"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L660","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dynamic_forms","data_retrieval_spec":[{"type":"metadata","id":"flexipage_metadata","params":{"types_and_members":{"FlexiPage":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"72ef1f8748d19297ded1a49d6d8df2e4faf2e42403dd62120748d5ef17da6f56"} {"id":"scuba:admin_007_005","source_id":"scuba","original_id":"admin_007_005","kind":"benchmark_task","instruction":"Create a dynamic form on the Case object where the Priority field is only visible if the Subject contains “Important”.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_005","query_templated":"Create a dynamic form on the Case object. The field Priority should be visible only when Subject's value contains Important.","query":"Create a dynamic form on the Case object where the Priority field is only visible if the Subject contains “Important”.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Case","field_name":"Record.Priority","field_2":"{!Record.Subject}","operator":"CONTAINS","value":"Important"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Case","field_name":"Priority","field_2":"Subject","operator":"contains","value":"Important"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/34","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Dynamic Forms","source_difficulty":"medium","template_id":"admin_007","template_name":"create_dynamic_forms","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/34"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/34"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_dynamic_forms"}],"source_schema_notes":[],"template_registry":{"name":"create_dynamic_forms","original":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_dynamic_forms","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCreate a dynamic form on the Case object where the Priority field is only visible if the Subject contains “Important”.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Dynamic Forms","task_id":"admin_007_005","query_templated":"Create a dynamic form on the Case object. The field Priority should be visible only when Subject's value contains Important.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCreate a dynamic form on the Case object where the Priority field is only visible if the Subject contains “Important”.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Case","field_name":"Record.Priority","field_2":"{!Record.Subject}","operator":"CONTAINS","value":"Important"},"difficulty":"medium","query_template_name":"create_dynamic_forms","query_template_metadata":{"template_id":"admin_007","template_string":"Create a dynamic form on the {object_name} object. The field {field_name} should be visible only when {field_2}'s value {operator} {value}.","prerequisites":{},"objects":[],"metadata_types":["FlexiPage"]},"instance_dict":{"object_name":"Case","field_name":"Priority","field_2":"Subject","operator":"contains","value":"Important"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/34"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L660","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_dynamic_forms","data_retrieval_spec":[{"type":"metadata","id":"flexipage_metadata","params":{"types_and_members":{"FlexiPage":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c1744b4e06039e4a3aa60e002888609550ef68d6731ce1a44ebf7bd4f6d57d2a"} {"id":"scuba:admin_008_001","source_id":"scuba","original_id":"admin_008_001","kind":"benchmark_task","instruction":"Create a list view on the Opportunity object named High Probability Opportunities that shows records where Stage = Proposal/Price Quote and Probability (%) ≥ 50, and make it visible to All Users.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_001","query_templated":"Create a list view of records of the Opportunity object. The list is named as High Probability Opportunities. Select records whose Stage is equals Proposal/Price Quote and whose Probability (%) is greater or equal 50. All Users should be able to see the list view.","query":"Create a list view on the Opportunity object named High Probability Opportunities that shows records where Stage = Proposal/Price Quote and Probability (%) ≥ 50, and make it visible to All Users.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Opportunity","list_name":"High Probability Opportunities","key1":"OPPORTUNITY.STAGE_NAME","key2":"OPPORTUNITY.PROBABILITY","operator1":"equals","operator2":"greaterOrEqual","value1":"Proposal/Price Quote","value2":"50.0"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Opportunity","list_name":"High Probability Opportunities","key1":"Stage","key2":"Probability (%)","operator1":"equals","operator2":"greater or equal","value1":"Proposal/Price Quote","value2":50},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/35","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/35"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/35"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate a list view on the Opportunity object named High Probability Opportunities that shows records where Stage = Proposal/Price Quote and Probability (%) ≥ 50, and make it visible to All Users.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_001","query_templated":"Create a list view of records of the Opportunity object. The list is named as High Probability Opportunities. Select records whose Stage is equals Proposal/Price Quote and whose Probability (%) is greater or equal 50. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate a list view on the Opportunity object named High Probability Opportunities that shows records where Stage = Proposal/Price Quote and Probability (%) ≥ 50, and make it visible to All Users.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Opportunity","list_name":"High Probability Opportunities","key1":"OPPORTUNITY.STAGE_NAME","key2":"OPPORTUNITY.PROBABILITY","operator1":"equals","operator2":"greaterOrEqual","value1":"Proposal/Price Quote","value2":"50.0"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Opportunity","list_name":"High Probability Opportunities","key1":"Stage","key2":"Probability (%)","operator1":"equals","operator2":"greater or equal","value1":"Proposal/Price Quote","value2":50},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/35"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5fa6cb81b0e0d4a1ace302711e29661450b7d51c25859d9ccfc390687f581bc9"} {"id":"scuba:admin_008_002","source_id":"scuba","original_id":"admin_008_002","kind":"benchmark_task","instruction":"On the Contact object, set up a public list view named Executive Decision Makers that shows only records with Title exactly CEO and Lead Source set to Web.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_002","query_templated":"Create a list view of records of the Contact object. The list is named as Executive Decision Makers. Select records whose Title is equals CEO and whose Lead Source is equals Web. All Users should be able to see the list view.","query":"On the Contact object, set up a public list view named Executive Decision Makers that shows only records with Title exactly CEO and Lead Source set to Web.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Contact","list_name":"Executive Decision Makers","key1":"TITLE","key2":"LEAD_SOURCE","operator1":"equals","value1":"CEO","operator2":"equals","value2":"Web"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Contact","list_name":"Executive Decision Makers","key1":"Title","key2":"Lead Source","operator1":"equals","value1":"CEO","operator2":"equals","value2":"Web"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/36","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/36"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/36"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nOn the Contact object, set up a public list view named Executive Decision Makers that shows only records with Title exactly CEO and Lead Source set to Web.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_002","query_templated":"Create a list view of records of the Contact object. The list is named as Executive Decision Makers. Select records whose Title is equals CEO and whose Lead Source is equals Web. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nOn the Contact object, set up a public list view named Executive Decision Makers that shows only records with Title exactly CEO and Lead Source set to Web.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Contact","list_name":"Executive Decision Makers","key1":"TITLE","key2":"LEAD_SOURCE","operator1":"equals","value1":"CEO","operator2":"equals","value2":"Web"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Contact","list_name":"Executive Decision Makers","key1":"Title","key2":"Lead Source","operator1":"equals","value1":"CEO","operator2":"equals","value2":"Web"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/36"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"68ed77e20089b9c375b53011c38212f5bda4f382d27c015cc04ad858e3ae8b20"} {"id":"scuba:admin_008_003","source_id":"scuba","original_id":"admin_008_003","kind":"benchmark_task","instruction":"Create a list view on the Task object named Overdue Follow-Ups that shows records where Status = Not Started and Due Date < Today, and make it visible to All Users.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_003","query_templated":"Create a list view of records of the Task object. The list is named as Overdue Follow-Ups. Select records whose Status is equals Not Started and whose Due Date is less than today. All Users should be able to see the list view.","query":"Create a list view on the Task object named Overdue Follow-Ups that shows records where Status = Not Started and Due Date < Today, and make it visible to All Users.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Task","list_name":"Overdue Follow-Ups","key1":"Status","operator1":"equals","value1":"Not Started","key2":"Due Date","operator2":"less than","value2":"today"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Task","list_name":"Overdue Follow-Ups","key1":"Status","operator1":"equals","value1":"Not Started","key2":"Due Date","operator2":"less than","value2":"today"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/37","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/37"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/37"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate a list view on the Task object named Overdue Follow-Ups that shows records where Status = Not Started and Due Date < Today, and make it visible to All Users.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_003","query_templated":"Create a list view of records of the Task object. The list is named as Overdue Follow-Ups. Select records whose Status is equals Not Started and whose Due Date is less than today. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate a list view on the Task object named Overdue Follow-Ups that shows records where Status = Not Started and Due Date < Today, and make it visible to All Users.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Task","list_name":"Overdue Follow-Ups","key1":"Status","operator1":"equals","value1":"Not Started","key2":"Due Date","operator2":"less than","value2":"today"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Task","list_name":"Overdue Follow-Ups","key1":"Status","operator1":"equals","value1":"Not Started","key2":"Due Date","operator2":"less than","value2":"today"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/37"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"646b4bf38a25dfc38d9b0e723b8f031834bfe584dccb4474210b109d85150365"} {"id":"scuba:admin_008_004","source_id":"scuba","original_id":"admin_008_004","kind":"benchmark_task","instruction":"I need a Case list view called Escalated Support Cases that filters strictly to Priority = High and Status = Escalated. Make sure the view is available to all users.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_004","query_templated":"Create a list view of records of the Case object. The list is named as Escalated Support Cases. Select records whose Priority is equals High and whose Status is equals Escalated. All Users should be able to see the list view.","query":"I need a Case list view called Escalated Support Cases that filters strictly to Priority = High and Status = Escalated. Make sure the view is available to all users.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","list_name":"Escalated Support Cases","key1":"CASES.PRIORITY","operator1":"equals","value1":"High","key2":"CASES.STATUS","operator2":"equals","value2":"Escalated"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Case","list_name":"Escalated Support Cases","key1":"Priority","operator1":"equals","value1":"High","key2":"Status","operator2":"equals","value2":"Escalated"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/38","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/38"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/38"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nI need a Case list view called Escalated Support Cases that filters strictly to Priority = High and Status = Escalated. Make sure the view is available to all users.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_004","query_templated":"Create a list view of records of the Case object. The list is named as Escalated Support Cases. Select records whose Priority is equals High and whose Status is equals Escalated. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nI need a Case list view called Escalated Support Cases that filters strictly to Priority = High and Status = Escalated. Make sure the view is available to all users.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","list_name":"Escalated Support Cases","key1":"CASES.PRIORITY","operator1":"equals","value1":"High","key2":"CASES.STATUS","operator2":"equals","value2":"Escalated"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Case","list_name":"Escalated Support Cases","key1":"Priority","operator1":"equals","value1":"High","key2":"Status","operator2":"equals","value2":"Escalated"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/38"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"308b80d67d0341ea68ea12d649f92160076b01923df29663c814ee4e056c737b"} {"id":"scuba:admin_008_005","source_id":"scuba","original_id":"admin_008_005","kind":"benchmark_task","instruction":"Create an Account list view named Filtered Accounts. Apply filters for Billing State/Province equals New York and Type not equal to Prospect, and share the view with all users.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_005","query_templated":"Create a list view of records of the Account object. The list is named as Filtered Accounts. Select records whose Billing State/Province is equals New York and whose Type is not equal to Prospect. All Users should be able to see the list view.","query":"Create an Account list view named Filtered Accounts. Apply filters for Billing State/Province equals New York and Type not equal to Prospect, and share the view with all users.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","list_name":"Filtered Accounts","key1":"ACCOUNT.ADDRESS1_STATE_CODE","operator1":"equals","value1":"NY","key2":"ACCOUNT.TYPE","operator2":"notEqual","value2":"Prospect"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Account","list_name":"Filtered Accounts","key1":"Billing State/Province","operator1":"equals","value1":"New York","key2":"Type","operator2":"not equal to","value2":"Prospect"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/39","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/39"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/39"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate an Account list view named Filtered Accounts. Apply filters for Billing State/Province equals New York and Type not equal to Prospect, and share the view with all users.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_005","query_templated":"Create a list view of records of the Account object. The list is named as Filtered Accounts. Select records whose Billing State/Province is equals New York and whose Type is not equal to Prospect. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nCreate an Account list view named Filtered Accounts. Apply filters for Billing State/Province equals New York and Type not equal to Prospect, and share the view with all users.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","list_name":"Filtered Accounts","key1":"ACCOUNT.ADDRESS1_STATE_CODE","operator1":"equals","value1":"NY","key2":"ACCOUNT.TYPE","operator2":"notEqual","value2":"Prospect"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Account","list_name":"Filtered Accounts","key1":"Billing State/Province","operator1":"equals","value1":"New York","key2":"Type","operator2":"not equal to","value2":"Prospect"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/39"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"1fb23af31af95719215fa6c8da99b30c7f56c0b95198a09f6d656b75bb894e10"} {"id":"scuba:admin_008_006","source_id":"scuba","original_id":"admin_008_006","kind":"benchmark_task","instruction":"Let’s add a Lead list view named Marketing Qualified Leads that only returns records where Lead Status is Open - Not Contacted and Rating is Hot. Set the visibility to all users.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_006","query_templated":"Create a list view of records of the Lead object. The list is named as Marketing Qualified Leads. Select records whose Lead Status is equals Open - Not Contacted and whose Rating is equals Hot. All Users should be able to see the list view.","query":"Let’s add a Lead list view named Marketing Qualified Leads that only returns records where Lead Status is Open - Not Contacted and Rating is Hot. Set the visibility to all users.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","list_name":"Marketing Qualified Leads","key1":"LEAD.STATUS","operator1":"equals","value1":"Open - Not Contacted","key2":"LEAD.RATING","operator2":"equals","value2":"Hot"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Lead","list_name":"Marketing Qualified Leads","key1":"Lead Status","operator1":"equals","value1":"Open - Not Contacted","key2":"Rating","operator2":"equals","value2":"Hot"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/40","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"easy","template_id":"admin_008","template_name":"create_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/40"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/40"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view","original":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nLet’s add a Lead list view named Marketing Qualified Leads that only returns records where Lead Status is Open - Not Contacted and Rating is Hot. Set the visibility to all users.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_008_006","query_templated":"Create a list view of records of the Lead object. The list is named as Marketing Qualified Leads. Select records whose Lead Status is equals Open - Not Contacted and whose Rating is equals Hot. All Users should be able to see the list view.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view.\n\n## Your ultimate task\n```\nLet’s add a Lead list view named Marketing Qualified Leads that only returns records where Lead Status is Open - Not Contacted and Rating is Hot. Set the visibility to all users.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","list_name":"Marketing Qualified Leads","key1":"LEAD.STATUS","operator1":"equals","value1":"Open - Not Contacted","key2":"LEAD.RATING","operator2":"equals","value2":"Hot"},"difficulty":"easy","query_template_name":"create_list_view","query_template_metadata":{"template_id":"admin_008","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_name}. Select records whose {key1} is {operator1} {value1} and whose {key2} is {operator2} {value2}. All Users should be able to see the list view.","prerequisites":{},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Lead","list_name":"Marketing Qualified Leads","key1":"Lead Status","operator1":"equals","value1":"Open - Not Contacted","key2":"Rating","operator2":"equals","value2":"Hot"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Contact] object. The list is named as [Executive Decision Makers]. Select records whose [Title] is [CEO] and whose [Lead Source] is [Web]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Contacts,” and open the Contacts object.\n- Open the list view controls/settings and select “Create New View.”\n- Enter the view name “Executive Decision Makers” and set visibility to “All users can see this view.” Save.\n- Open the Filters panel, add filter: Field = Title, Operator = equals, Value = CEO. Confirm/Done.\n- Add another filter: Field = Lead Source, Operator = equals, Value = Web (select the corresponding picklist value). Confirm/Done.\n- Save the list view.\nSample 2:\nTASK:Create a list view of records of the [Task] object. The list is named as [Overdue Follow-Ups]. Select records whose [Status] is [Not Started] and whose [Due Date] is [less than] [Today]. [All users] should be able to see the list view.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Tasks,” and open the Tasks object.\n- Switch the list display to “Table.”\n- Open List View Controls and select “New.”\n- Enter list name “Overdue Follow-Ups,” set visibility to “All users can see this list view,” and Save.\n- Add Filter: Field = Status, Value = Not Started, then Done.\n- Add Filter: Field = Due Date, Operator = less than, Value = Today, then Done.\n- Save the list view."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/40"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L402","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a6fef84cbae5a9dc8720f9dc60bd4f99eeee6d82279cc1622a3025757422505a"} {"id":"scuba:admin_009_001","source_id":"scuba","original_id":"admin_009_001","kind":"benchmark_task","instruction":"Please create a new list view on the Opportunity object named ByAmount. Show only two columns—Account Name and Amount—and make it visible exclusively to the \"CFO\" and the \"Channel Sales Team\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_001","query_templated":"Create a list view of records of the Opportunity object. The list is named as ByAmount. Include only Account Name and Amount field. The list view should only be accessible to 'CFO' and 'Channel Sales Team'.","query":"Please create a new list view on the Opportunity object named ByAmount. Show only two columns—Account Name and Amount—and make it visible exclusively to the \"CFO\" and the \"Channel Sales Team\".","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","list_view_name":"ByAmount","field1":"ACCOUNT.NAME","field2":"OPPORTUNITY.AMOUNT","users":["CFO","ChannelSalesTeam"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Opportunity","list_view_name":"ByAmount","field1":"Account Name","field2":"Amount","users":"'CFO' and 'Channel Sales Team'"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/41","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"medium","template_id":"admin_009","template_name":"create_list_view_share","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/41"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/41"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view_share"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view_share","original":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a new list view on the Opportunity object named ByAmount. Show only two columns—Account Name and Amount—and make it visible exclusively to the \"CFO\" and the \"Channel Sales Team\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_001","query_templated":"Create a list view of records of the Opportunity object. The list is named as ByAmount. Include only Account Name and Amount field. The list view should only be accessible to 'CFO' and 'Channel Sales Team'.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a new list view on the Opportunity object named ByAmount. Show only two columns—Account Name and Amount—and make it visible exclusively to the \"CFO\" and the \"Channel Sales Team\".\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","list_view_name":"ByAmount","field1":"ACCOUNT.NAME","field2":"OPPORTUNITY.AMOUNT","users":["CFO","ChannelSalesTeam"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Opportunity","list_view_name":"ByAmount","field1":"Account Name","field2":"Amount","users":"'CFO' and 'Channel Sales Team'"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/41"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L445","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view_share","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_view_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"ba538d3ccaafcbe7568f9902ff00ac89c57cf78651b1ad8c532c36df9bc36eff"} {"id":"scuba:admin_009_002","source_id":"scuba","original_id":"admin_009_002","kind":"benchmark_task","instruction":"Please create a list view for the Case object called AllCasesByPriority with exactly these columns: Case Number and Priority. Limit access so only \"Customer Support, International\", \"Project Lead\", and \"Customer Support, North America\" can see it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_002","query_templated":"Create a list view of records of the Case object. The list is named as AllCasesByPriority. Include only Case Number and Priority field. The list view should only be accessible to 'Customer Support, International', 'Project Lead' and 'Customer Support, North America'.","query":"Please create a list view for the Case object called AllCasesByPriority with exactly these columns: Case Number and Priority. Limit access so only \"Customer Support, International\", \"Project Lead\", and \"Customer Support, North America\" can see it.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","list_view_name":"AllCasesByPriority","field1":"CASES.CASE_NUMBER","field2":"CASES.PRIORITY","users":["CustomerSupportInternational","Project_Lead","CustomerSupportNorthAmerica"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Case","list_view_name":"AllCasesByPriority","field1":"Case Number","field2":"Priority","users":"'Customer Support, International', 'Project Lead' and 'Customer Support, North America'"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/42","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"medium","template_id":"admin_009","template_name":"create_list_view_share","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/42"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/42"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view_share"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view_share","original":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a list view for the Case object called AllCasesByPriority with exactly these columns: Case Number and Priority. Limit access so only \"Customer Support, International\", \"Project Lead\", and \"Customer Support, North America\" can see it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_002","query_templated":"Create a list view of records of the Case object. The list is named as AllCasesByPriority. Include only Case Number and Priority field. The list view should only be accessible to 'Customer Support, International', 'Project Lead' and 'Customer Support, North America'.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a list view for the Case object called AllCasesByPriority with exactly these columns: Case Number and Priority. Limit access so only \"Customer Support, International\", \"Project Lead\", and \"Customer Support, North America\" can see it.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","list_view_name":"AllCasesByPriority","field1":"CASES.CASE_NUMBER","field2":"CASES.PRIORITY","users":["CustomerSupportInternational","Project_Lead","CustomerSupportNorthAmerica"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Case","list_view_name":"AllCasesByPriority","field1":"Case Number","field2":"Priority","users":"'Customer Support, International', 'Project Lead' and 'Customer Support, North America'"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/42"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L445","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view_share","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_view_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b2029cccb1683e2724b13f6ca060c95312de1dc1813a92ce0bd5e33c6cce243b"} {"id":"scuba:admin_009_003","source_id":"scuba","original_id":"admin_009_003","kind":"benchmark_task","instruction":"Could you set up an Account list view named Sales Accounts that displays only the Name and Industry fields, and restrict its visibility to just the \"Account Avengers\" and the \"Lead Legends\"?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_003","query_templated":"Create a list view of records of the Account object. The list is named as Sales Accounts. Include only Name and Industry field. The list view should only be accessible to 'Account Avengers' and 'Lead Legends'.","query":"Could you set up an Account list view named Sales Accounts that displays only the Name and Industry fields, and restrict its visibility to just the \"Account Avengers\" and the \"Lead Legends\"?","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","list_view_name":"Sales Accounts","field1":"ACCOUNT.NAME","field2":"ACCOUNT.INDUSTRY","users":["Account_Avengers","Lead_Legends"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Account","list_view_name":"Sales Accounts","field1":"Name","field2":"Industry","users":"'Account Avengers' and 'Lead Legends'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/43","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"medium","template_id":"admin_009","template_name":"create_list_view_share","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/43"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/43"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view_share"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view_share","original":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you set up an Account list view named Sales Accounts that displays only the Name and Industry fields, and restrict its visibility to just the \"Account Avengers\" and the \"Lead Legends\"?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_003","query_templated":"Create a list view of records of the Account object. The list is named as Sales Accounts. Include only Name and Industry field. The list view should only be accessible to 'Account Avengers' and 'Lead Legends'.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you set up an Account list view named Sales Accounts that displays only the Name and Industry fields, and restrict its visibility to just the \"Account Avengers\" and the \"Lead Legends\"?\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","list_view_name":"Sales Accounts","field1":"ACCOUNT.NAME","field2":"ACCOUNT.INDUSTRY","users":["Account_Avengers","Lead_Legends"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Account","list_view_name":"Sales Accounts","field1":"Name","field2":"Industry","users":"'Account Avengers' and 'Lead Legends'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/43"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L445","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view_share","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_view_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4527100d90bc5376120bbef0774bb12c2fc2b01fba867505d67038becc7be3d1"} {"id":"scuba:admin_009_004","source_id":"scuba","original_id":"admin_009_004","kind":"benchmark_task","instruction":"On the Contact object, build a list view titled Support Contacts. The columns should be strictly FirstName and Email—no others. Share it only with \"Contact Crusaders\" and \"Case Commandos\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_004","query_templated":"Create a list view of records of the Contact object. The list is named as Support Contacts. Include only FirstName and Email field. The list view should only be accessible to 'Contact Crusaders' and 'Case Commandos'.","query":"On the Contact object, build a list view titled Support Contacts. The columns should be strictly FirstName and Email—no others. Share it only with \"Contact Crusaders\" and \"Case Commandos\".","human_review_comment":"accept","ground_truth_dict":{"object_name":"Contact","list_view_name":"Support Contacts","field1":"CONTACT.FIRST_NAME","field2":"CONTACT.EMAIL","users":["Contact_Crusaders","Case_Commandos"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Contact","list_view_name":"Support Contacts","field1":"FirstName","field2":"Email","users":"'Contact Crusaders' and 'Case Commandos'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/44","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"medium","template_id":"admin_009","template_name":"create_list_view_share","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/44"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/44"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view_share"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view_share","original":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nOn the Contact object, build a list view titled Support Contacts. The columns should be strictly FirstName and Email—no others. Share it only with \"Contact Crusaders\" and \"Case Commandos\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_004","query_templated":"Create a list view of records of the Contact object. The list is named as Support Contacts. Include only FirstName and Email field. The list view should only be accessible to 'Contact Crusaders' and 'Case Commandos'.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nOn the Contact object, build a list view titled Support Contacts. The columns should be strictly FirstName and Email—no others. Share it only with \"Contact Crusaders\" and \"Case Commandos\".\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Contact","list_view_name":"Support Contacts","field1":"CONTACT.FIRST_NAME","field2":"CONTACT.EMAIL","users":["Contact_Crusaders","Case_Commandos"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Contact","list_view_name":"Support Contacts","field1":"FirstName","field2":"Email","users":"'Contact Crusaders' and 'Case Commandos'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/44"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L445","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view_share","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_view_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"aaff1e4dcbfea90c7422e21ed6e410db7d775533bc149e87ab05198dee70e98e"} {"id":"scuba:admin_009_005","source_id":"scuba","original_id":"admin_009_005","kind":"benchmark_task","instruction":"For the Lead object, I need a minimal list view called Qualified Leads that includes just Company and Phone. Lock it down so only \"Lead Legends\" and \"Opportunity Oracles\" have access.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_005","query_templated":"Create a list view of records of the Lead object. The list is named as Qualified Leads. Include only Company and Phone field. The list view should only be accessible to 'Lead Legends' and 'Opportunity Oracles'.","query":"For the Lead object, I need a minimal list view called Qualified Leads that includes just Company and Phone. Lock it down so only \"Lead Legends\" and \"Opportunity Oracles\" have access.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","list_view_name":"Qualified Leads","field1":"LEAD.COMPANY","field2":"LEAD.PHONE","users":["Lead_Legends","Opportunity_Oracles"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Lead","list_view_name":"Qualified Leads","field1":"Company","field2":"Phone","users":"'Lead Legends' and 'Opportunity Oracles'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/45","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"medium","template_id":"admin_009","template_name":"create_list_view_share","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/45"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/45"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_list_view_share"}],"source_schema_notes":[],"template_registry":{"name":"create_list_view_share","original":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_list_view_share","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the Lead object, I need a minimal list view called Qualified Leads that includes just Company and Phone. Lock it down so only \"Lead Legends\" and \"Opportunity Oracles\" have access.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_009_005","query_templated":"Create a list view of records of the Lead object. The list is named as Qualified Leads. Include only Company and Phone field. The list view should only be accessible to 'Lead Legends' and 'Opportunity Oracles'.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the Lead object, I need a minimal list view called Qualified Leads that includes just Company and Phone. Lock it down so only \"Lead Legends\" and \"Opportunity Oracles\" have access.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","list_view_name":"Qualified Leads","field1":"LEAD.COMPANY","field2":"LEAD.PHONE","users":["Lead_Legends","Opportunity_Oracles"]},"difficulty":"medium","query_template_name":"create_list_view_share","query_template_metadata":{"template_id":"admin_009","template_string":"Create a list view of records of the {object_name} object. The list is named as {list_view_name}. Include only {field1} and {field2} field. The list view should only be accessible to {users}.","prerequisites":{"data":{"objects":["Group"]}},"objects":[],"metadata_types":["ListView"]},"instance_dict":{"object_name":"Lead","list_view_name":"Qualified Leads","field1":"Company","field2":"Phone","users":"'Lead Legends' and 'Opportunity Oracles'"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a list view of records of the [Opportunity] object. The list is named as [ByAmount]. Include only [Account Name] and [Amount] field. The list view should only be accsible to [CFO] and [Channel Sales Team].\nEXPERIENCE:\nSuccessful plan to create the Opportunity list view “ByAmount” with only Account Name and Amount, shared to CFO and Channel Sales Team:\n\n- Access Opportunities\n - Log in.\n - Open App Launcher, search “opportunities,” and open the Opportunities object.\n\n- Create the list view\n - Open List View Controls > New.\n - Name the list “ByAmount.”\n - Set visibility to “Share list view with group of users.”\n - Choose Roles, search and add “CFO” and “Channel Sales Team.”\n - Save.\n\n- Configure visible fields\n - Open List View Controls > Select Fields to Display.\n - Keep only Account Name and Amount by removing other default columns (removed: Opportunity Name, Stage, Close Date, Opportunity Owner Alias).\n - Save.\nSample 2:\nTASK:Create a list view of records of the [Case] object. The list is named as [AllCasesByPriority]. Include only [Case Number] and [Priority] fields. The list view should only be accessible to [Customer Support, International], [Project Lead], and [Customer Support, North America].\nEXPERIENCE:\nSuccessful plan to create the AllCasesByPriority list view for Case:\n\n1) Open Cases\n- Log in.\n- Open App Launcher, search for “cases”, and select “Cases”.\n\n2) Create the list view\n- Open List View Controls > New.\n- Set List Name: AllCasesByPriority.\n- In “Who sees this list view,” choose “Share list view with groups of users.”\n- For sharing, choose Roles and add:\n - Customer Support, International\n - Project Lead\n - Customer Support, North America\n- Save.\n\n3) Configure visible fields\n- Open List View Controls > Select Fields to Display.\n- Remove all fields except Case Number and Priority (removed Contact Name, Subject, Status, Case Owner Alias).\n- Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/45"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L445","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_list_view_share","data_retrieval_spec":[{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id, DeveloperName, Name, SObjectType FROM ListView WHERE Name='{list_view_name}'"}},{"type":"metadata","id":"list_view_metadata","params":{"types_and_members":{"ListView":["{object_name}.{retrieved_results['list_view_info'].records[0]['DeveloperName']}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"476c062c43b5f942da19dab32214c70c353c3e8ea67a2bfc61ac6cf74fe4227b"} {"id":"scuba:admin_010_001","source_id":"scuba","original_id":"admin_010_001","kind":"benchmark_task","instruction":"Build a donut chart titled \"By Amount\" on the Opportunity object’s All Opportunities list view. Sum the Amount field and group the slices by Account.Name.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_001","query_templated":"Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.","query":"Build a donut chart titled \"By Amount\" on the Opportunity object’s All Opportunities list view. Sum the Amount field and group the slices by Account.Name.","human_review_comment":"Change spin up","ground_truth_dict":{"object_name":"Opportunity","chart_name":"By Amount","list_name":"All Opportunities","aggregate_field":"Opportunity.Amount","aggregate_type":"Sum","grouping_field":"Opportunity.AccountId","chart_type":"pie"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Opportunity","chart_name":"By Amount","list_name":"All Opportunities","aggregate_field":"Amount","aggregate_type":"Sum","grouping_field":"Account.Name","chart_type":"Donut"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/46","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"hard","template_id":"admin_010","template_name":"create_chart_for_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/46"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/46"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_chart_for_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_chart_for_list_view","original":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nBuild a donut chart titled \"By Amount\" on the Opportunity object’s All Opportunities list view. Sum the Amount field and group the slices by Account.Name.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_001","query_templated":"Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nBuild a donut chart titled \"By Amount\" on the Opportunity object’s All Opportunities list view. Sum the Amount field and group the slices by Account.Name.\n```","human_review_comment":"Change spin up","ground_truth_dict":{"object_name":"Opportunity","chart_name":"By Amount","list_name":"All Opportunities","aggregate_field":"Opportunity.Amount","aggregate_type":"Sum","grouping_field":"Opportunity.AccountId","chart_type":"pie"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Opportunity","chart_name":"By Amount","list_name":"All Opportunities","aggregate_field":"Amount","aggregate_type":"Sum","grouping_field":"Account.Name","chart_type":"Donut"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/46"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L771","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_chart_for_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_chart_info","params":{"query":"SELECT Id, AggregateType, AggregateField, GroupingField, ChartType FROM ListViewChart WHERE MasterLabel='{chart_name}'"}},{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id FROM ListView WHERE Name='{list_name}'"}},{"type":"soql","id":"chart_instance_info","params":{"query":"SELECT Label FROM ListViewChartInstance WHERE ListViewContextId='{retrieved_results['list_view_info'].records[0]['Id']}' AND SourceEntity='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"774ee5bcf9d6497b24763f77f1dfa214e86a23be783ac35767bb25e4b6138508"} {"id":"scuba:admin_010_002","source_id":"scuba","original_id":"admin_010_002","kind":"benchmark_task","instruction":"Create a Vertical Bar chart named Accounts by Revenue on the Accounts object under the All Accounts list view, aggregated on Annual Revenue (Sum) and grouped by Account Name.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_002","query_templated":"Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.","query":"Create a Vertical Bar chart named Accounts by Revenue on the Accounts object under the All Accounts list view, aggregated on Annual Revenue (Sum) and grouped by Account Name.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Account","chart_name":"Accounts by Revenue","list_name":"All Accounts","aggregate_field":"Account.AnnualRevenue","aggregate_type":"Sum","grouping_field":"Account.Name","chart_type":"vbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Accounts","chart_name":"Accounts by Revenue","list_name":"All Accounts","aggregate_field":"Annual Revenue","aggregate_type":"Sum","grouping_field":"Account Name","chart_type":"Vertical Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/47","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"hard","template_id":"admin_010","template_name":"create_chart_for_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/47"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/47"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_chart_for_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_chart_for_list_view","original":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nCreate a Vertical Bar chart named Accounts by Revenue on the Accounts object under the All Accounts list view, aggregated on Annual Revenue (Sum) and grouped by Account Name.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_002","query_templated":"Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nCreate a Vertical Bar chart named Accounts by Revenue on the Accounts object under the All Accounts list view, aggregated on Annual Revenue (Sum) and grouped by Account Name.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Account","chart_name":"Accounts by Revenue","list_name":"All Accounts","aggregate_field":"Account.AnnualRevenue","aggregate_type":"Sum","grouping_field":"Account.Name","chart_type":"vbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Accounts","chart_name":"Accounts by Revenue","list_name":"All Accounts","aggregate_field":"Annual Revenue","aggregate_type":"Sum","grouping_field":"Account Name","chart_type":"Vertical Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/47"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L771","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_chart_for_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_chart_info","params":{"query":"SELECT Id, AggregateType, AggregateField, GroupingField, ChartType FROM ListViewChart WHERE MasterLabel='{chart_name}'"}},{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id FROM ListView WHERE Name='{list_name}'"}},{"type":"soql","id":"chart_instance_info","params":{"query":"SELECT Label FROM ListViewChartInstance WHERE ListViewContextId='{retrieved_results['list_view_info'].records[0]['Id']}' AND SourceEntity='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0a4f20f6afbaecf3a6e8b67f759210d75f776403e9d5f80b272d41c2f28fab14"} {"id":"scuba:admin_010_003","source_id":"scuba","original_id":"admin_010_003","kind":"benchmark_task","instruction":"I want to visualize how many lead sources are tied to converted opportunities. Build a Vertical Bar chart called \"Number of Sources for converted opportunities\" on the Leads object under the All Open Leads list view. Use Count on the Lead Source field and group the results by Converted Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_003","query_templated":"Create a Vertical Bar chart named Number of Sources for converted opportunities for the Leads object, under the All Open Leads list view. Aggregated on the Lead Source field by Count. The grouping field should be Converted Opportunity.","query":"I want to visualize how many lead sources are tied to converted opportunities. Build a Vertical Bar chart called \"Number of Sources for converted opportunities\" on the Leads object under the All Open Leads list view. Use Count on the Lead Source field and group the results by Converted Opportunity.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","chart_name":"Number of Sources for converted opportunities","list_name":"All Open Leads","aggregate_field":"Lead.LeadSource","aggregate_type":"Count","grouping_field":"Lead.ConvertedOpportunityId","chart_type":"vbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Leads","chart_name":"Number of Sources for converted opportunities","list_name":"All Open Leads","aggregate_field":"Lead Source","aggregate_type":"Count","grouping_field":"Converted Opportunity","chart_type":"Vertical Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/48","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"hard","template_id":"admin_010","template_name":"create_chart_for_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/48"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/48"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_chart_for_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_chart_for_list_view","original":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nI want to visualize how many lead sources are tied to converted opportunities. Build a Vertical Bar chart called \"Number of Sources for converted opportunities\" on the Leads object under the All Open Leads list view. Use Count on the Lead Source field and group the results by Converted Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_003","query_templated":"Create a Vertical Bar chart named Number of Sources for converted opportunities for the Leads object, under the All Open Leads list view. Aggregated on the Lead Source field by Count. The grouping field should be Converted Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nI want to visualize how many lead sources are tied to converted opportunities. Build a Vertical Bar chart called \"Number of Sources for converted opportunities\" on the Leads object under the All Open Leads list view. Use Count on the Lead Source field and group the results by Converted Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","chart_name":"Number of Sources for converted opportunities","list_name":"All Open Leads","aggregate_field":"Lead.LeadSource","aggregate_type":"Count","grouping_field":"Lead.ConvertedOpportunityId","chart_type":"vbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Leads","chart_name":"Number of Sources for converted opportunities","list_name":"All Open Leads","aggregate_field":"Lead Source","aggregate_type":"Count","grouping_field":"Converted Opportunity","chart_type":"Vertical Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/48"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L771","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_chart_for_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_chart_info","params":{"query":"SELECT Id, AggregateType, AggregateField, GroupingField, ChartType FROM ListViewChart WHERE MasterLabel='{chart_name}'"}},{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id FROM ListView WHERE Name='{list_name}'"}},{"type":"soql","id":"chart_instance_info","params":{"query":"SELECT Label FROM ListViewChartInstance WHERE ListViewContextId='{retrieved_results['list_view_info'].records[0]['Id']}' AND SourceEntity='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"fca9dfb7284411bb33050c01d77d32c54175f15eccf77a3e7c13251ccb79b156"} {"id":"scuba:admin_010_004","source_id":"scuba","original_id":"admin_010_004","kind":"benchmark_task","instruction":"For Cases → All Open Cases, add a Horizontal Bar chart named \"Accounts by Reason.\" Count using the Account Name field as the aggregate, and group the chart by Case Reason.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_004","query_templated":"Create a Horizontal Bar chart named Accounts by Reason for the Cases object, under the All Open Cases list view. Aggregated on the Account Name field by Count. The grouping field should be Case Reason.","query":"For Cases → All Open Cases, add a Horizontal Bar chart named \"Accounts by Reason.\" Count using the Account Name field as the aggregate, and group the chart by Case Reason.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","chart_name":"Accounts by Reason","list_name":"All Open Cases","aggregate_field":"Case.AccountId","aggregate_type":"Count","grouping_field":"Case.Reason","chart_type":"hbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Cases","chart_name":"Accounts by Reason","list_name":"All Open Cases","aggregate_field":"Account Name","aggregate_type":"Count","grouping_field":"Case Reason","chart_type":"Horizontal Bar"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/49","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"hard","template_id":"admin_010","template_name":"create_chart_for_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/49"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/49"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_chart_for_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_chart_for_list_view","original":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nFor Cases → All Open Cases, add a Horizontal Bar chart named \"Accounts by Reason.\" Count using the Account Name field as the aggregate, and group the chart by Case Reason.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_004","query_templated":"Create a Horizontal Bar chart named Accounts by Reason for the Cases object, under the All Open Cases list view. Aggregated on the Account Name field by Count. The grouping field should be Case Reason.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nFor Cases → All Open Cases, add a Horizontal Bar chart named \"Accounts by Reason.\" Count using the Account Name field as the aggregate, and group the chart by Case Reason.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","chart_name":"Accounts by Reason","list_name":"All Open Cases","aggregate_field":"Case.AccountId","aggregate_type":"Count","grouping_field":"Case.Reason","chart_type":"hbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Cases","chart_name":"Accounts by Reason","list_name":"All Open Cases","aggregate_field":"Account Name","aggregate_type":"Count","grouping_field":"Case Reason","chart_type":"Horizontal Bar"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/49"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L771","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_chart_for_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_chart_info","params":{"query":"SELECT Id, AggregateType, AggregateField, GroupingField, ChartType FROM ListViewChart WHERE MasterLabel='{chart_name}'"}},{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id FROM ListView WHERE Name='{list_name}'"}},{"type":"soql","id":"chart_instance_info","params":{"query":"SELECT Label FROM ListViewChartInstance WHERE ListViewContextId='{retrieved_results['list_view_info'].records[0]['Id']}' AND SourceEntity='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"70d9116b13a001bcd5bffcae978f869717eee34bfd832bc87d2dd7b01750c25a"} {"id":"scuba:admin_010_005","source_id":"scuba","original_id":"admin_010_005","kind":"benchmark_task","instruction":"Create a Horizontal Bar chart called \"Average orders by Type\" on the Order object under the All Orders list view. Calculate the Average of Order Amount and group by Order Type.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_005","query_templated":"Create a Horizontal Bar chart chart named Average orders by Type for the Order object, under the All Orders list view. Aggregated on the Order Amount field by Average. The grouping field should be Order Type.","query":"Create a Horizontal Bar chart called \"Average orders by Type\" on the Order object under the All Orders list view. Calculate the Average of Order Amount and group by Order Type.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Order","chart_name":"Average orders by Type","list_name":"All Orders","aggregate_field":"Order.TotalAmount","aggregate_type":"Avg","grouping_field":"Order.Type","chart_type":"hbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Order","chart_name":"Average orders by Type","list_name":"All Orders","aggregate_field":"Order Amount","aggregate_type":"Average","grouping_field":"Order Type","chart_type":"Horizontal Bar chart"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/50","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"List view","source_difficulty":"hard","template_id":"admin_010","template_name":"create_chart_for_list_view","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/50"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/50"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_chart_for_list_view"}],"source_schema_notes":[],"template_registry":{"name":"create_chart_for_list_view","original":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_chart_for_list_view","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nCreate a Horizontal Bar chart called \"Average orders by Type\" on the Order object under the All Orders list view. Calculate the Average of Order Amount and group by Order Type.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"List view","task_id":"admin_010_005","query_templated":"Create a Horizontal Bar chart chart named Average orders by Type for the Order object, under the All Orders list view. Aggregated on the Order Amount field by Average. The grouping field should be Order Type.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart.\n\n## Your ultimate task\n```\nCreate a Horizontal Bar chart called \"Average orders by Type\" on the Order object under the All Orders list view. Calculate the Average of Order Amount and group by Order Type.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Order","chart_name":"Average orders by Type","list_name":"All Orders","aggregate_field":"Order.TotalAmount","aggregate_type":"Avg","grouping_field":"Order.Type","chart_type":"hbar"},"difficulty":"hard","query_template_name":"create_chart_for_list_view","query_template_metadata":{"template_id":"admin_010","template_string":"Create a {chart_type} chart named {chart_name} for the {object_name} object, under the {list_name} list view. Aggregated on the {aggregate_field} field by {aggregate_type}. The grouping field should be {grouping_field}.","prerequisites":{},"objects":["ListViewChart"],"metadata_types":[]},"instance_dict":{"object_name":"Order","chart_name":"Average orders by Type","list_name":"All Orders","aggregate_field":"Order Amount","aggregate_type":"Average","grouping_field":"Order Type","chart_type":"Horizontal Bar chart"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Donut chart named By Amount for the Opportunity object, under the All Opportunities list view. Aggregated on the Amount field by Sum. The grouping field should be Account.Name.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open the Opportunities app:\n - Click App Launcher, type “opportunities” in the search box, and select Opportunities.\n- Switch to the correct list view:\n - Open the list view selector and choose All Opportunities.\n- Create the chart:\n - Open Charts, click the Settings (gear) icon, then New Chart.\n - In Chart Name, enter “By Amount”.\n - Open the Chart Type dropdown and select Donut.\n - Open the Aggregate Type dropdown and select Sum. (Aggregate Field remained at the default Amount; Grouping Field remained at the default Account.Name.)\n - Click Save.\nSample 2:\nTASK:Create a Vertical Bar chart named Accounts by Revenue for the Accounts object, under the All Accounts list view. Aggregated on the Annual Revenue field by Sum. The grouping field should be Account Name.\nEXPERIENCE:\nSuccessful plan to create the chart:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Accounts,” and open the Accounts app/object.\n- Switch the list view to “All Accounts.”\n- Open the Charts panel, then open Chart Settings and select “New Chart.”\n- Set Chart Name to “Accounts by Revenue.”\n- Set Chart Type to “Vertical Bar Chart.”\n- Configure aggregation:\n - Aggregate Field: Annual Revenue\n - Aggregate Type: Sum\n- Set Grouping Field to “Account Name.”\n- Save the chart."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/50"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L771","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_chart_for_list_view","data_retrieval_spec":[{"type":"soql","id":"list_view_chart_info","params":{"query":"SELECT Id, AggregateType, AggregateField, GroupingField, ChartType FROM ListViewChart WHERE MasterLabel='{chart_name}'"}},{"type":"soql","id":"list_view_info","params":{"query":"SELECT Id FROM ListView WHERE Name='{list_name}'"}},{"type":"soql","id":"chart_instance_info","params":{"query":"SELECT Label FROM ListViewChartInstance WHERE ListViewContextId='{retrieved_results['list_view_info'].records[0]['Id']}' AND SourceEntity='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"30a711b3f9756b00eadea8218ff3535a647fb93c650d2f829afd432a3f759ae9"} {"id":"scuba:admin_011_001","source_id":"scuba","original_id":"admin_011_001","kind":"benchmark_task","instruction":"Spin up a new app named Energy Consultations. Add Task and Contact to the navigation bar from the available items list. In User Profiles, select System Administrator, Custom: Sales Profile from the available profiles. Keep all other required fields at their default values.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_001","query_templated":"Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.","query":"Spin up a new app named Energy Consultations. Add Task and Contact to the navigation bar from the available items list. In User Profiles, select System Administrator, Custom: Sales Profile from the available profiles. Keep all other required fields at their default values.","human_review_comment":"Change signal","ground_truth_dict":{"app_name":"Energy Consultations","comma_separated_objects_list":"Task, Contact","comma_separated_user_profiles":"Admin, Custom%3A Sales Profile"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Energy Consultations","comma_separated_objects_list":"Task, Contact","comma_separated_user_profiles":"System Administrator, Custom: Sales Profile"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/51","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Lightning App","source_difficulty":"hard","template_id":"admin_011","template_name":"create_app","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/51"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/51"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_app"}],"source_schema_notes":[],"template_registry":{"name":"create_app","original":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nSpin up a new app named Energy Consultations. Add Task and Contact to the navigation bar from the available items list. In User Profiles, select System Administrator, Custom: Sales Profile from the available profiles. Keep all other required fields at their default values.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_001","query_templated":"Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nSpin up a new app named Energy Consultations. Add Task and Contact to the navigation bar from the available items list. In User Profiles, select System Administrator, Custom: Sales Profile from the available profiles. Keep all other required fields at their default values.\n```","human_review_comment":"Change signal","ground_truth_dict":{"app_name":"Energy Consultations","comma_separated_objects_list":"Task, Contact","comma_separated_user_profiles":"Admin, Custom%3A Sales Profile"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Energy Consultations","comma_separated_objects_list":"Task, Contact","comma_separated_user_profiles":"System Administrator, Custom: Sales Profile"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/51"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L149","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_app","data_retrieval_spec":[{"type":"metadata","id":"applications_and_profiles","params":{"types_and_members":{"CustomApplication":["{app_name.replace(' ', '_')}"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7efa9d58062a42e157afcbab71985dff6f617b9dc8ce82bfbffa7b03d2d094c8"} {"id":"scuba:admin_011_002","source_id":"scuba","original_id":"admin_011_002","kind":"benchmark_task","instruction":"Create an app named Customer Service with navigation items Customers and Analytics, assign User Profile as Customer Portal Manager (Standard), Standard User, and Custom: Support Profile profiles, and leave all other required fields as default.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_002","query_templated":"Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.","query":"Create an app named Customer Service with navigation items Customers and Analytics, assign User Profile as Customer Portal Manager (Standard), Standard User, and Custom: Support Profile profiles, and leave all other required fields as default.","human_review_comment":"rewrite","ground_truth_dict":{"app_name":"Customer Service","comma_separated_objects_list":"Customer, WaveHomeLightning","comma_separated_user_profiles":"Customer Portal Manager Standard, Standard, Custom%3A Support Profile"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Customer Service","comma_separated_objects_list":"Customers, Analytics","comma_separated_user_profiles":"Customer Portal Manager (Standard), Standard User, Support User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/52","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Lightning App","source_difficulty":"hard","template_id":"admin_011","template_name":"create_app","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/52"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/52"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_app"}],"source_schema_notes":[],"template_registry":{"name":"create_app","original":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nCreate an app named Customer Service with navigation items Customers and Analytics, assign User Profile as Customer Portal Manager (Standard), Standard User, and Custom: Support Profile profiles, and leave all other required fields as default.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_002","query_templated":"Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nCreate an app named Customer Service with navigation items Customers and Analytics, assign User Profile as Customer Portal Manager (Standard), Standard User, and Custom: Support Profile profiles, and leave all other required fields as default.\n```","human_review_comment":"rewrite","ground_truth_dict":{"app_name":"Customer Service","comma_separated_objects_list":"Customer, WaveHomeLightning","comma_separated_user_profiles":"Customer Portal Manager Standard, Standard, Custom%3A Support Profile"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Customer Service","comma_separated_objects_list":"Customers, Analytics","comma_separated_user_profiles":"Customer Portal Manager (Standard), Standard User, Support User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/52"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L149","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_app","data_retrieval_spec":[{"type":"metadata","id":"applications_and_profiles","params":{"types_and_members":{"CustomApplication":["{app_name.replace(' ', '_')}"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"bf02f4c74b6b8238f6b5e6b2e8807a62a8b366e804f8b7f1d6de686f4713ba57"} {"id":"scuba:admin_011_003","source_id":"scuba","original_id":"admin_011_003","kind":"benchmark_task","instruction":"First, create an app named Administration. Next, add Case and Report to its navigation from the available items. Then, under User Profiles, choose Partner Community User and Analytics Cloud Integration User from the available profiles. Finally, leave all remaining required fields at their default settings.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_003","query_templated":"Create an app with name Administration. Choose Case, Report objects from available items as the navigation items. For User Profiles, choose the Partner Community User, Analytics Cloud Integration User from the available profiles. Leave the rest required fields as the default.","query":"First, create an app named Administration. Next, add Case and Report to its navigation from the available items. Then, under User Profiles, choose Partner Community User and Analytics Cloud Integration User from the available profiles. Finally, leave all remaining required fields at their default settings.","human_review_comment":"accept","ground_truth_dict":{"app_name":"Administration","comma_separated_objects_list":"Case, report","comma_separated_user_profiles":"Partner Community User, Analytics Cloud Integration User"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Administration","comma_separated_objects_list":"Case, Report","comma_separated_user_profiles":"Partner Community User, Analytics Cloud Integration User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/53","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Lightning App","source_difficulty":"hard","template_id":"admin_011","template_name":"create_app","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/53"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/53"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_app"}],"source_schema_notes":[],"template_registry":{"name":"create_app","original":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nFirst, create an app named Administration. Next, add Case and Report to its navigation from the available items. Then, under User Profiles, choose Partner Community User and Analytics Cloud Integration User from the available profiles. Finally, leave all remaining required fields at their default settings.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_003","query_templated":"Create an app with name Administration. Choose Case, Report objects from available items as the navigation items. For User Profiles, choose the Partner Community User, Analytics Cloud Integration User from the available profiles. Leave the rest required fields as the default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nFirst, create an app named Administration. Next, add Case and Report to its navigation from the available items. Then, under User Profiles, choose Partner Community User and Analytics Cloud Integration User from the available profiles. Finally, leave all remaining required fields at their default settings.\n```","human_review_comment":"accept","ground_truth_dict":{"app_name":"Administration","comma_separated_objects_list":"Case, report","comma_separated_user_profiles":"Partner Community User, Analytics Cloud Integration User"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Administration","comma_separated_objects_list":"Case, Report","comma_separated_user_profiles":"Partner Community User, Analytics Cloud Integration User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/53"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L149","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_app","data_retrieval_spec":[{"type":"metadata","id":"applications_and_profiles","params":{"types_and_members":{"CustomApplication":["{app_name.replace(' ', '_')}"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f56ba4653735d6c2c12165d9394d49ac5e07b61ebaacb3e9b15bf95f0d4edc3c"} {"id":"scuba:admin_011_004","source_id":"scuba","original_id":"admin_011_004","kind":"benchmark_task","instruction":"Please provision a new app titled Sales Customized App. For the nav items, include only Lead and Product from the available items. From the available profiles, assign Standard User and Contract Manager. Everything else should stay on the default required values.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_004","query_templated":"Create an app with name Sales Customized App. Choose Lead, Product objects from available items as the navigation items. For User Profiles, choose the Standard User, Contract Manager from the available profiles. Leave the rest required fields as the default.","query":"Please provision a new app titled Sales Customized App. For the nav items, include only Lead and Product from the available items. From the available profiles, assign Standard User and Contract Manager. Everything else should stay on the default required values.","human_review_comment":"accept","ground_truth_dict":{"app_name":"Sales Customized App","comma_separated_objects_list":"Lead, Product2","comma_separated_user_profiles":"Standard, ContractManager"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Sales Customized App","comma_separated_objects_list":"Lead, Product","comma_separated_user_profiles":"Standard User, Contract Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/54","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Lightning App","source_difficulty":"hard","template_id":"admin_011","template_name":"create_app","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/54"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/54"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_app"}],"source_schema_notes":[],"template_registry":{"name":"create_app","original":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nPlease provision a new app titled Sales Customized App. For the nav items, include only Lead and Product from the available items. From the available profiles, assign Standard User and Contract Manager. Everything else should stay on the default required values.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_004","query_templated":"Create an app with name Sales Customized App. Choose Lead, Product objects from available items as the navigation items. For User Profiles, choose the Standard User, Contract Manager from the available profiles. Leave the rest required fields as the default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nPlease provision a new app titled Sales Customized App. For the nav items, include only Lead and Product from the available items. From the available profiles, assign Standard User and Contract Manager. Everything else should stay on the default required values.\n```","human_review_comment":"accept","ground_truth_dict":{"app_name":"Sales Customized App","comma_separated_objects_list":"Lead, Product2","comma_separated_user_profiles":"Standard, ContractManager"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Sales Customized App","comma_separated_objects_list":"Lead, Product","comma_separated_user_profiles":"Standard User, Contract Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/54"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L149","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_app","data_retrieval_spec":[{"type":"metadata","id":"applications_and_profiles","params":{"types_and_members":{"CustomApplication":["{app_name.replace(' ', '_')}"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b54dadc4a34b7cb46646c205cf70b2f72340b31be088629122c588f7bc382fff"} {"id":"scuba:admin_011_005","source_id":"scuba","original_id":"admin_011_005","kind":"benchmark_task","instruction":"I’d like to roll out an app called Highly Customized Service App. Put Case and Incident on the navigation menu, picking them from the available items list. In User Profiles, select Solution Manager and Minimum Access - Salesforce from the available profiles. Leave the rest of the required fields untouched—use the defaults.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_005","query_templated":"Create an app with name Highly Customized Service App. Choose Case, Incident objects from available items as the navigation items. For User Profiles, choose the Solution Manager, Minimum Access - Salesforce from the available profiles. Leave the rest required fields as the default.","query":"I’d like to roll out an app called Highly Customized Service App. Put Case and Incident on the navigation menu, picking them from the available items list. In User Profiles, select Solution Manager and Minimum Access - Salesforce from the available profiles. Leave the rest of the required fields untouched—use the defaults.","human_review_comment":"accept","ground_truth_dict":{"app_name":"Highly Customized Service App","comma_separated_objects_list":"Case, Incident","comma_separated_user_profiles":"SolutionManager, Minimum Access - Salesforce"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Highly Customized Service App","comma_separated_objects_list":"Case, Incident","comma_separated_user_profiles":"Solution Manager, Minimum Access - Salesforce"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/55","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Lightning App","source_difficulty":"hard","template_id":"admin_011","template_name":"create_app","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/55"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/55"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_app"}],"source_schema_notes":[],"template_registry":{"name":"create_app","original":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_app","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nI’d like to roll out an app called Highly Customized Service App. Put Case and Incident on the navigation menu, picking them from the available items list. In User Profiles, select Solution Manager and Minimum Access - Salesforce from the available profiles. Leave the rest of the required fields untouched—use the defaults.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Lightning App","task_id":"admin_011_005","query_templated":"Create an app with name Highly Customized Service App. Choose Case, Incident objects from available items as the navigation items. For User Profiles, choose the Solution Manager, Minimum Access - Salesforce from the available profiles. Leave the rest required fields as the default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app.\n\n## Your ultimate task\n```\nI’d like to roll out an app called Highly Customized Service App. Put Case and Incident on the navigation menu, picking them from the available items list. In User Profiles, select Solution Manager and Minimum Access - Salesforce from the available profiles. Leave the rest of the required fields untouched—use the defaults.\n```","human_review_comment":"accept","ground_truth_dict":{"app_name":"Highly Customized Service App","comma_separated_objects_list":"Case, Incident","comma_separated_user_profiles":"SolutionManager, Minimum Access - Salesforce"},"difficulty":"hard","query_template_name":"create_app","query_template_metadata":{"template_id":"admin_011","template_string":"Create an app with name {app_name}. Choose {comma_separated_objects_list} objects from available items as the navigation items. For User Profiles, choose the {comma_separated_user_profiles} from the available profiles. Leave the rest required fields as the default.","prerequisites":{},"objects":[],"metadata_types":["CustomApplication"]},"instance_dict":{"app_name":"Highly Customized Service App","comma_separated_objects_list":"Case, Incident","comma_separated_user_profiles":"Solution Manager, Minimum Access - Salesforce"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an app with name Energy Consultations. Choose Task, Contact objects from available items as the navigation items. For User Profiles, choose the System Administrator, Custom: Sales Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan to create the Lightning app:\n\n- Log in and open Setup: Click Setup (gear) -> Setup. In Quick Find, type “App Manager” and open App Manager.\n- Create the app: Click New Lightning App. Enter App Name: “Energy Consultations”. Leave other required fields at defaults (Developer Name auto-populates). Click Next through the intermediate setup pages until Navigation Items.\n- Configure Navigation Items: In Available Items search, type “Task”, select “Tasks”, click Add. Then type “Contact”, select “Contacts”, click Add. Click Next.\n- Assign User Profiles: In Available Profiles search, type “System Administrator”, select it, click Add. Then type “Custom: Sales Profile”, select it, click Add.\n- Finish: Click Save & Finish.\nSample 2:\nTASK:Create an app with name Customer Service. Choose Customers, Analytics objects from available items as the navigation items. For User Profiles, choose the Customer Portal Manager (Standard), Standard User, Custom: Support Profile from the available profiles. Leave the rest required fields as the default.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup (gear icon > Setup).\n- In Quick Find, type “App Manager” and open App Manager.\n- Click “New Lightning App.”\n- App Details & Branding:\n - Set App Name to “Customer Service.”\n - Leave Developer Name as the auto-populated default.\n - Click Next.\n- Proceed through the next configuration screens (App Options, Utility Items, etc.) keeping defaults and clicking Next until Navigation Items.\n- Navigation Items:\n - In Available Items, search “Customers,” select it, and click Add.\n - Search “Analytics,” select it, and click Add.\n - Click Next.\n- User Profiles:\n - In Available Profiles, search and add each of:\n - Customer Portal Manager (Standard)\n - Standard User\n - Custom: Support Profile\n - Click Save & Finish to create the app."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/55"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L149","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_app","data_retrieval_spec":[{"type":"metadata","id":"applications_and_profiles","params":{"types_and_members":{"CustomApplication":["{app_name.replace(' ', '_')}"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f74a0f3ca2f856d423dff55f3bfe1438dd0455f3f74ed02373b4e09a4fb5fbae"} {"id":"scuba:admin_012_001","source_id":"scuba","original_id":"admin_012_001","kind":"benchmark_task","instruction":"Create a custom tab for the MyAnimal object, use the Sun tab style, restrict visibility to the System Administrator profile only, and make it available exclusively inside the Sales App.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_001","query_templated":"Create a custom tab for a custom object MyAnimal. Choose the Sun tab style. Make the tab visible to only System Administrator profile and can only be accssed through the Sales App.","query":"Create a custom tab for the MyAnimal object, use the Sun tab style, restrict visibility to the System Administrator profile only, and make it available exclusively inside the Sales App.","human_review_comment":"change spin up","ground_truth_dict":{"object_name":"MyAnimal","tab_style":"Custom3: Sun","profile_name":"Admin","app_name":"standard__Sales"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyAnimal","tab_style":"Sun","profile_name":"System Administrator","app_name":"Sales"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/56","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Tab","source_difficulty":"hard","template_id":"admin_012","template_name":"create_custom_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/56"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/56"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_tab"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_tab","original":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nCreate a custom tab for the MyAnimal object, use the Sun tab style, restrict visibility to the System Administrator profile only, and make it available exclusively inside the Sales App.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_001","query_templated":"Create a custom tab for a custom object MyAnimal. Choose the Sun tab style. Make the tab visible to only System Administrator profile and can only be accssed through the Sales App.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nCreate a custom tab for the MyAnimal object, use the Sun tab style, restrict visibility to the System Administrator profile only, and make it available exclusively inside the Sales App.\n```","human_review_comment":"change spin up","ground_truth_dict":{"object_name":"MyAnimal","tab_style":"Custom3: Sun","profile_name":"Admin","app_name":"standard__Sales"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyAnimal","tab_style":"Sun","profile_name":"System Administrator","app_name":"Sales"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/56"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L359","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_tabs_and_profiles","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"Profile":["*"]}}},{"type":"metadata","id":"custom_applications","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"CustomApplication":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"aac8ce309a97eee5575324cdd144834f458c718caa6084814d36193c0d06b99b"} {"id":"scuba:admin_012_002","source_id":"scuba","original_id":"admin_012_002","kind":"benchmark_task","instruction":"I need a MyBike custom tab using the Books style; hide it from everyone except the External Apps User profile, and surface it only in the Platform App.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_002","query_templated":"Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.","query":"I need a MyBike custom tab using the Books style; hide it from everyone except the External Apps User profile, and surface it only in the Platform App.","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyBike","tab_style":"Custom55: Books","profile_name":"External Apps Login User","app_name":"standard__Platform"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyBike","tab_style":"Books","profile_name":"External Apps User","app_name":"Platform"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/57","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Tab","source_difficulty":"hard","template_id":"admin_012","template_name":"create_custom_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/57"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/57"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_tab"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_tab","original":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nI need a MyBike custom tab using the Books style; hide it from everyone except the External Apps User profile, and surface it only in the Platform App.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_002","query_templated":"Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nI need a MyBike custom tab using the Books style; hide it from everyone except the External Apps User profile, and surface it only in the Platform App.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyBike","tab_style":"Custom55: Books","profile_name":"External Apps Login User","app_name":"standard__Platform"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyBike","tab_style":"Books","profile_name":"External Apps User","app_name":"Platform"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/57"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L359","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_tabs_and_profiles","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"Profile":["*"]}}},{"type":"metadata","id":"custom_applications","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"CustomApplication":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7241985c7acc91fccbfba45373508159fa8ad404d0dfdfc11083dbd86297a125"} {"id":"scuba:admin_012_003","source_id":"scuba","original_id":"admin_012_003","kind":"benchmark_task","instruction":"Add a custom tab for MyVehicle, pick the Truck tab style, and lock it so it’s visible to the Marketing User profile only—accessible solely via the Marketing CRM Classsic App.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_003","query_templated":"Create a custom tab for a custom object MyVehicle. Choose the Truck tab style. Make the tab visible to only Marketing User profile and can only be accssed through the Marketing CRM Classsic App.","query":"Add a custom tab for MyVehicle, pick the Truck tab style, and lock it so it’s visible to the Marketing User profile only—accessible solely via the Marketing CRM Classsic App.","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyVehicle","tab_style":"Custom98: Truck","profile_name":"MarketingProfile","app_name":"standard__Marketing"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyVehicle","tab_style":"Truck","profile_name":"Marketing User","app_name":"Marketing CRM Classsic"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/58","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Tab","source_difficulty":"hard","template_id":"admin_012","template_name":"create_custom_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/58"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/58"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_tab"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_tab","original":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nAdd a custom tab for MyVehicle, pick the Truck tab style, and lock it so it’s visible to the Marketing User profile only—accessible solely via the Marketing CRM Classsic App.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_003","query_templated":"Create a custom tab for a custom object MyVehicle. Choose the Truck tab style. Make the tab visible to only Marketing User profile and can only be accssed through the Marketing CRM Classsic App.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nAdd a custom tab for MyVehicle, pick the Truck tab style, and lock it so it’s visible to the Marketing User profile only—accessible solely via the Marketing CRM Classsic App.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyVehicle","tab_style":"Custom98: Truck","profile_name":"MarketingProfile","app_name":"standard__Marketing"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyVehicle","tab_style":"Truck","profile_name":"Marketing User","app_name":"Marketing CRM Classsic"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/58"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L359","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_tabs_and_profiles","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"Profile":["*"]}}},{"type":"metadata","id":"custom_applications","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"CustomApplication":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"055c877e61696281316563abb9c41134ebe4dcdb81db8c7c951b2c2ddae1cb28"} {"id":"scuba:admin_012_004","source_id":"scuba","original_id":"admin_012_004","kind":"benchmark_task","instruction":"Do this for Environment: create a custom tab, style it with Books, grant visibility solely to the Solution Manager profile, and constrain it to the Service App.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_004","query_templated":"Create a custom tab for a custom object Environment. Choose the Books tab style. Make the tab visible to only Solution Manager profile and can only be accssed through the Service App.","query":"Do this for Environment: create a custom tab, style it with Books, grant visibility solely to the Solution Manager profile, and constrain it to the Service App.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Environment","tab_style":"Custom55: Books","profile_name":"SolutionManager","app_name":"standard__Service"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"Environment","tab_style":"Books","profile_name":"Solution Manager","app_name":"Service"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/59","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Tab","source_difficulty":"hard","template_id":"admin_012","template_name":"create_custom_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/59"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/59"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_tab"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_tab","original":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nDo this for Environment: create a custom tab, style it with Books, grant visibility solely to the Solution Manager profile, and constrain it to the Service App.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_004","query_templated":"Create a custom tab for a custom object Environment. Choose the Books tab style. Make the tab visible to only Solution Manager profile and can only be accssed through the Service App.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nDo this for Environment: create a custom tab, style it with Books, grant visibility solely to the Solution Manager profile, and constrain it to the Service App.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Environment","tab_style":"Custom55: Books","profile_name":"SolutionManager","app_name":"standard__Service"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"Environment","tab_style":"Books","profile_name":"Solution Manager","app_name":"Service"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/59"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L359","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_tabs_and_profiles","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"Profile":["*"]}}},{"type":"metadata","id":"custom_applications","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"CustomApplication":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"93021acc425e44d38b821055512c2dcc2dccfcfad8b3a995c161bd0450a29d89"} {"id":"scuba:admin_012_005","source_id":"scuba","original_id":"admin_012_005","kind":"benchmark_task","instruction":"Please provision a custom tab for MyInventory with the Box look, visible only to the Contract Manager profile and available exclusively in the Approvals App.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_005","query_templated":"Create a custom tab for a custom object MyInventory. Choose the Box tab style. Make the tab visible to only Contract Manager profile and can only be accssed through the Approvals App.","query":"Please provision a custom tab for MyInventory with the Box look, visible only to the Contract Manager profile and available exclusively in the Approvals App.","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyInventory","tab_style":"Custom13: Box","profile_name":"ContractManager","app_name":"standard__Approvals"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyInventory","tab_style":"Box","profile_name":"Contract Manager","app_name":"Approvals"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/60","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Tab","source_difficulty":"hard","template_id":"admin_012","template_name":"create_custom_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/60"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/60"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_tab"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_tab","original":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nPlease provision a custom tab for MyInventory with the Box look, visible only to the Contract Manager profile and available exclusively in the Approvals App.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Tab","task_id":"admin_012_005","query_templated":"Create a custom tab for a custom object MyInventory. Choose the Box tab style. Make the tab visible to only Contract Manager profile and can only be accssed through the Approvals App.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save.\n\n## Your ultimate task\n```\nPlease provision a custom tab for MyInventory with the Box look, visible only to the Contract Manager profile and available exclusively in the Approvals App.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyInventory","tab_style":"Custom13: Box","profile_name":"ContractManager","app_name":"standard__Approvals"},"difficulty":"hard","query_template_name":"create_custom_tab","query_template_metadata":{"template_id":"admin_012","template_string":"Create a custom tab for a custom object {object_name}. Choose the {tab_style} tab style. Make the tab visible to only {profile_name} profile and can only be accssed through the {app_name} App.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":[],"metadata_types":["CustomTab"]},"instance_dict":{"object_name":"MyInventory","tab_style":"Box","profile_name":"Contract Manager","app_name":"Approvals"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom tab for a custom object [MyAnimal]. Choose the [Sun] tab style. Make the tab visible to only [System Administrator] profile and can only be accssed through the [Sales] App.\nEXPERIENCE:\nSuccessful plan:\n\n- Authenticate:\n - Click Log In, enter the verification code, and click Verify.\n\n- Navigate to tab creation:\n - Go to Setup > Create > Tabs (Custom Tabs).\n - Start creating a new custom object tab.\n\n- Configure tab:\n - Select the custom object: MyAnimal.\n - Open Tab Style picker and choose Sun.\n - Click Next.\n\n- Set profile visibility:\n - Set “Tab Visibility for all profiles” to Default Off.\n - For the System Administrator profile, set Tab Visibility to Default On.\n - Click Next.\n\n- Add to apps:\n - Use the “Select/Deselect All” checkbox to clear all apps.\n - Check only the Sales app.\n - Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom tab for a custom object MyBike. Choose the Books tab style. Make the tab visible to only External Apps User profile and can only be accssed through the Platform App.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Tabs” and open the Tabs page.\n- Under Custom Object Tabs, click New.\n- Set Object to “MyBike”.\n- Click Tab Style and choose “Books”.\n- Click Next.\n- Profile visibility:\n - Set “Tab Hidden” for all profiles.\n - Change the “External Apps User” profile to “Default On”.\n- Click Next.\n- App selection:\n - Ensure only “Platform App” is checked (uncheck any “Select All” or other apps).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/60"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L359","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_tabs_and_profiles","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"Profile":["*"]}}},{"type":"metadata","id":"custom_applications","params":{"types_and_members":{"CustomTab":["{object_name}__c"],"CustomApplication":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7a29dd16ccd9afdcb9434838c63b710841abcd4259cb27f1dfec8ba1c2e31dd0"} {"id":"scuba:admin_014_001","source_id":"scuba","original_id":"admin_014_001","kind":"benchmark_task","instruction":"Please set up a rule on the Contact object called \"Contact with Fuzzy Logic.\" For the Email field, require an Exact match, and make sure the Match Blank Field option stays off.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_001","query_templated":"Create a matching rule named Contact with Fuzzy Logic on the Contact. Use Exact matching method for the Email field and leave its Match Blank Field box unchecked.","query":"Please set up a rule on the Contact object called \"Contact with Fuzzy Logic.\" For the Email field, require an Exact match, and make sure the Match Blank Field option stays off.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Contact with Fuzzy Logic","object_name":"Contact","match_method":"Exact","field_name":"Email"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Contact with Fuzzy Logic","object_name":"Contact","match_method":"Exact matching","field_name":"Email"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/61","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Deduplication","source_difficulty":"medium","template_id":"admin_014","template_name":"create_matching_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/61"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/61"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matching_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_matching_rule","original":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nPlease set up a rule on the Contact object called \"Contact with Fuzzy Logic.\" For the Email field, require an Exact match, and make sure the Match Blank Field option stays off.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_001","query_templated":"Create a matching rule named Contact with Fuzzy Logic on the Contact. Use Exact matching method for the Email field and leave its Match Blank Field box unchecked.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nPlease set up a rule on the Contact object called \"Contact with Fuzzy Logic.\" For the Email field, require an Exact match, and make sure the Match Blank Field option stays off.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Contact with Fuzzy Logic","object_name":"Contact","match_method":"Exact","field_name":"Email"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Contact with Fuzzy Logic","object_name":"Contact","match_method":"Exact matching","field_name":"Email"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/61"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L314","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matching_rule","data_retrieval_spec":[{"type":"metadata","id":"matching_rule_metadata","params":{"types_and_members":{"MatchingRule":["{object_name}.{rule_name.replace(' ', '_')}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2840f6151e0d86455c801cf126ba213a57f6beef527b06d6779d1c0cc4b6487d"} {"id":"scuba:admin_014_002","source_id":"scuba","original_id":"admin_014_002","kind":"benchmark_task","instruction":"Build a Lead matching rule named \"Lead Phone Match\" where the Phone field uses the Fuzzy method. Also, don’t count empty values—leave the Match Blank Field box unchecked.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_002","query_templated":"Create a matching rule named Lead Phone Match on the Lead. Use Fuzzy matching method for the Phone field and leave its Match Blank Field box unchecked.","query":"Build a Lead matching rule named \"Lead Phone Match\" where the Phone field uses the Fuzzy method. Also, don’t count empty values—leave the Match Blank Field box unchecked.","human_review_comment":"delete ending things","ground_truth_dict":{"rule_name":"Lead Phone Match","object_name":"Lead","match_method":"Phone","field_name":"Phone"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Lead Phone Match","object_name":"Lead","match_method":"Fuzzy matching","field_name":"Phone"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/62","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Deduplication","source_difficulty":"medium","template_id":"admin_014","template_name":"create_matching_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/62"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/62"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matching_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_matching_rule","original":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nBuild a Lead matching rule named \"Lead Phone Match\" where the Phone field uses the Fuzzy method. Also, don’t count empty values—leave the Match Blank Field box unchecked.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_002","query_templated":"Create a matching rule named Lead Phone Match on the Lead. Use Fuzzy matching method for the Phone field and leave its Match Blank Field box unchecked.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nBuild a Lead matching rule named \"Lead Phone Match\" where the Phone field uses the Fuzzy method. Also, don’t count empty values—leave the Match Blank Field box unchecked.\n```","human_review_comment":"delete ending things","ground_truth_dict":{"rule_name":"Lead Phone Match","object_name":"Lead","match_method":"Phone","field_name":"Phone"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Lead Phone Match","object_name":"Lead","match_method":"Fuzzy matching","field_name":"Phone"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/62"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L314","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matching_rule","data_retrieval_spec":[{"type":"metadata","id":"matching_rule_metadata","params":{"types_and_members":{"MatchingRule":["{object_name}.{rule_name.replace(' ', '_')}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"ea4bd4ee2677419e3024363644ee71949a6f2fb2be942f80b2da2d44db029d9f"} {"id":"scuba:admin_014_003","source_id":"scuba","original_id":"admin_014_003","kind":"benchmark_task","instruction":"Could you add an Account matching rule titled \"Account Website Match\"? Configure Website to match exactly, and keep the Match Blank Field checkbox unselected.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_003","query_templated":"Create a matching rule named Account Website Match on the Account. Use Exact matching method for the Website field and leave its Match Blank Field box unchecked.","query":"Could you add an Account matching rule titled \"Account Website Match\"? Configure Website to match exactly, and keep the Match Blank Field checkbox unselected.","human_review_comment":"adding matching","ground_truth_dict":{"rule_name":"Account Website Match","object_name":"Account","match_method":"Exact","field_name":"Website"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Account Website Match","object_name":"Account","match_method":"Exact matching","field_name":"Website"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/63","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Deduplication","source_difficulty":"medium","template_id":"admin_014","template_name":"create_matching_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/63"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/63"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matching_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_matching_rule","original":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nCould you add an Account matching rule titled \"Account Website Match\"? Configure Website to match exactly, and keep the Match Blank Field checkbox unselected.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_003","query_templated":"Create a matching rule named Account Website Match on the Account. Use Exact matching method for the Website field and leave its Match Blank Field box unchecked.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nCould you add an Account matching rule titled \"Account Website Match\"? Configure Website to match exactly, and keep the Match Blank Field checkbox unselected.\n```","human_review_comment":"adding matching","ground_truth_dict":{"rule_name":"Account Website Match","object_name":"Account","match_method":"Exact","field_name":"Website"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Account Website Match","object_name":"Account","match_method":"Exact matching","field_name":"Website"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/63"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L314","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matching_rule","data_retrieval_spec":[{"type":"metadata","id":"matching_rule_metadata","params":{"types_and_members":{"MatchingRule":["{object_name}.{rule_name.replace(' ', '_')}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e59bce3b1dd5577bef2bcf62884309b14e9974956f8e52a52084db5177fdb591"} {"id":"scuba:admin_014_004","source_id":"scuba","original_id":"admin_014_004","kind":"benchmark_task","instruction":"Create a rule called \"Address matching\" on the Address object: set the Address field to use Fuzzy matching, and do not tick the Match Blank Field box.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_004","query_templated":"Create a matching rule named Address matching on the Address. Use Fuzzy matching method for the Address field and leave its Match Blank Field box unchecked.","query":"Create a rule called \"Address matching\" on the Address object: set the Address field to use Fuzzy matching, and do not tick the Match Blank Field box.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Address matching","object_name":"Address","match_method":"Street","field_name":"Street"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Address matching","object_name":"Address","match_method":"Fuzzy matching","field_name":"Address"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/64","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Deduplication","source_difficulty":"medium","template_id":"admin_014","template_name":"create_matching_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/64"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/64"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matching_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_matching_rule","original":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nCreate a rule called \"Address matching\" on the Address object: set the Address field to use Fuzzy matching, and do not tick the Match Blank Field box.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_004","query_templated":"Create a matching rule named Address matching on the Address. Use Fuzzy matching method for the Address field and leave its Match Blank Field box unchecked.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nCreate a rule called \"Address matching\" on the Address object: set the Address field to use Fuzzy matching, and do not tick the Match Blank Field box.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Address matching","object_name":"Address","match_method":"Street","field_name":"Street"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Address matching","object_name":"Address","match_method":"Fuzzy matching","field_name":"Address"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/64"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L314","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matching_rule","data_retrieval_spec":[{"type":"metadata","id":"matching_rule_metadata","params":{"types_and_members":{"MatchingRule":["{object_name}.{rule_name.replace(' ', '_')}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c6ecb077a3458ebd4a505f13074b1bd25a0302c6bf2f7ba2dbf62efbeb6c60ac"} {"id":"scuba:admin_014_005","source_id":"scuba","original_id":"admin_014_005","kind":"benchmark_task","instruction":"For Individuals, add a matching rule named \"Individual Last Name Match.\" Use Fuzzy matching on the Last Name field, and ensure the Match Blank Field setting remains unchecked.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_005","query_templated":"Create a matching rule named Individual Last Name Match on the Individual. Use Fuzzy matching method for the Last Name field and leave its Match Blank Field box unchecked.","query":"For Individuals, add a matching rule named \"Individual Last Name Match.\" Use Fuzzy matching on the Last Name field, and ensure the Match Blank Field setting remains unchecked.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Individual Last Name Match","object_name":"Individual","match_method":"LastName","field_name":"LastName"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Individual Last Name Match","object_name":"Individual","match_method":"Fuzzy matching","field_name":"Last Name"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/65","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Deduplication","source_difficulty":"medium","template_id":"admin_014","template_name":"create_matching_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/65"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/65"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_matching_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_matching_rule","original":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_matching_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nFor Individuals, add a matching rule named \"Individual Last Name Match.\" Use Fuzzy matching on the Last Name field, and ensure the Match Blank Field setting remains unchecked.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Deduplication","task_id":"admin_014_005","query_templated":"Create a matching rule named Individual Last Name Match on the Individual. Use Fuzzy matching method for the Last Name field and leave its Match Blank Field box unchecked.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save.\n\n## Your ultimate task\n```\nFor Individuals, add a matching rule named \"Individual Last Name Match.\" Use Fuzzy matching on the Last Name field, and ensure the Match Blank Field setting remains unchecked.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Individual Last Name Match","object_name":"Individual","match_method":"LastName","field_name":"LastName"},"difficulty":"medium","query_template_name":"create_matching_rule","query_template_metadata":{"template_id":"admin_014","template_string":"Create a matching rule named {rule_name} on the {object_name}. Use {match_method} method for the {field_name} field and leave its Match Blank Field box unchecked.","prerequisites":{},"objects":[],"metadata_types":["MatchingRule"]},"instance_dict":{"rule_name":"Individual Last Name Match","object_name":"Individual","match_method":"Fuzzy matching","field_name":"Last Name"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a matching rule named [Contact with Fuzzy Logic] on the [Contact object]. Use [Exact macthing] method for the [email field] and leave its Match Black Field box unchecked.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Go to Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Select Object: Contact, then click Next.\n- Enter Rule Name: “Contact with Fuzzy Logic.”\n- Add matching criteria:\n - Field: Email\n - Matching Method: Exact\n - Ensure “Match Blank Fields” is unchecked\n- Click Save. If prompted for a unique name, set it to “FuzzyLogics” and Save again.\nSample 2:\nTASK:Create a matching rule named [Lead Phone Match] on the [Lead object]. Use the [Fuzzy matching] method for the [Phone field] and leave its Match Blank Field box unchecked. \nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Matching Rules” and open it.\n- Click “New Rule.”\n- Choose Object = Lead, then click Next.\n- Enter Rule Name = “Lead Phone Match.”\n- Define the matching criteria:\n - Field = Phone\n - Matching Method = Fuzzy\n - Ensure “Match Blank Field” remains unchecked (default).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/65"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L314","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_matching_rule","data_retrieval_spec":[{"type":"metadata","id":"matching_rule_metadata","params":{"types_and_members":{"MatchingRule":["{object_name}.{rule_name.replace(' ', '_')}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"644ad41e05b38b686515698ec9629ba72e21e83db0c355d6b02f349f21e8ece1"} {"id":"scuba:admin_016_001","source_id":"scuba","original_id":"admin_016_001","kind":"benchmark_task","instruction":"Please build a new custom object labeled MyHouse. Make sure the “Launch New Custom Tab Wizard after saving this custom object” box is enabled so you can choose any tab style you like, and leave everything else at the default settings.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_001","query_templated":"Create a custom object with label MyHouse. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"Please build a new custom object labeled MyHouse. Make sure the “Launch New Custom Tab Wizard after saving this custom object” box is enabled so you can choose any tab style you like, and leave everything else at the default settings.","human_review_comment":"change spin up","ground_truth_dict":{"custom_obj_name":"MyHouse"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"MyHouse"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/66","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"easy","template_id":"admin_016","template_name":"custom_obj_with_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/66"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/66"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/custom_obj_with_tab"}],"source_schema_notes":[],"template_registry":{"name":"custom_obj_with_tab","original":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nPlease build a new custom object labeled MyHouse. Make sure the “Launch New Custom Tab Wizard after saving this custom object” box is enabled so you can choose any tab style you like, and leave everything else at the default settings.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_001","query_templated":"Create a custom object with label MyHouse. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nPlease build a new custom object labeled MyHouse. Make sure the “Launch New Custom Tab Wizard after saving this custom object” box is enabled so you can choose any tab style you like, and leave everything else at the default settings.\n```","human_review_comment":"change spin up","ground_truth_dict":{"custom_obj_name":"MyHouse"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"MyHouse"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/66"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L274","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_custom_obj_with_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_tabs","params":{"types_and_members":{"CustomTab":["{custom_obj_name}__c"],"CustomObject":["{custom_obj_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f20d2ecc4e2a9604858473299dcb01526b501b11ca21ecd88910df106af50f62"} {"id":"scuba:admin_016_002","source_id":"scuba","original_id":"admin_016_002","kind":"benchmark_task","instruction":"Set up a custom object called WildBird. When you save it, tick the “Launch New Custom Tab Wizard after saving this custom object” option to pick whatever tab style you prefer. Don’t change any other fields—stick with the defaults.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_002","query_templated":"Create a custom object with label WildBird. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"Set up a custom object called WildBird. When you save it, tick the “Launch New Custom Tab Wizard after saving this custom object” option to pick whatever tab style you prefer. Don’t change any other fields—stick with the defaults.","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"WildBird"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"WildBird"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/67","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"easy","template_id":"admin_016","template_name":"custom_obj_with_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/67"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/67"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/custom_obj_with_tab"}],"source_schema_notes":[],"template_registry":{"name":"custom_obj_with_tab","original":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nSet up a custom object called WildBird. When you save it, tick the “Launch New Custom Tab Wizard after saving this custom object” option to pick whatever tab style you prefer. Don’t change any other fields—stick with the defaults.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_002","query_templated":"Create a custom object with label WildBird. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nSet up a custom object called WildBird. When you save it, tick the “Launch New Custom Tab Wizard after saving this custom object” option to pick whatever tab style you prefer. Don’t change any other fields—stick with the defaults.\n```","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"WildBird"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"WildBird"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/67"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L274","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_custom_obj_with_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_tabs","params":{"types_and_members":{"CustomTab":["{custom_obj_name}__c"],"CustomObject":["{custom_obj_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4eb41735e80496dd3517bfac4eddd951cef433e1793ee5bed7e8acba4094215b"} {"id":"scuba:admin_016_003","source_id":"scuba","original_id":"admin_016_003","kind":"benchmark_task","instruction":"Create a custom object with the label StormTracker. Turn on the “Launch New Custom Tab Wizard after saving this custom object” checkbox so a tab style can be selected, and keep all remaining settings at their default values.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_003","query_templated":"Create a custom object with label StormTracker. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"Create a custom object with the label StormTracker. Turn on the “Launch New Custom Tab Wizard after saving this custom object” checkbox so a tab style can be selected, and keep all remaining settings at their default values.","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"StormTracker"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"StormTracker"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/68","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"easy","template_id":"admin_016","template_name":"custom_obj_with_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/68"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/68"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/custom_obj_with_tab"}],"source_schema_notes":[],"template_registry":{"name":"custom_obj_with_tab","original":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nCreate a custom object with the label StormTracker. Turn on the “Launch New Custom Tab Wizard after saving this custom object” checkbox so a tab style can be selected, and keep all remaining settings at their default values.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_003","query_templated":"Create a custom object with label StormTracker. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nCreate a custom object with the label StormTracker. Turn on the “Launch New Custom Tab Wizard after saving this custom object” checkbox so a tab style can be selected, and keep all remaining settings at their default values.\n```","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"StormTracker"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"StormTracker"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/68"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L274","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_custom_obj_with_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_tabs","params":{"types_and_members":{"CustomTab":["{custom_obj_name}__c"],"CustomObject":["{custom_obj_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b7512f214e2c3fc2d06a51c399e132bae749f0688b69849ae45fe2484adc3d4d"} {"id":"scuba:admin_016_004","source_id":"scuba","original_id":"admin_016_004","kind":"benchmark_task","instruction":"Build a custom object named Bike, and have the New Custom Tab Wizard auto-launch on save by checking the “Launch New Custom Tab Wizard after saving this custom object” option. Choose any tab style you want and leave the rest as-is (defaults only).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_004","query_templated":"Create a custom object with label Bike. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"Build a custom object named Bike, and have the New Custom Tab Wizard auto-launch on save by checking the “Launch New Custom Tab Wizard after saving this custom object” option. Choose any tab style you want and leave the rest as-is (defaults only).","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"Bike"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"Bike"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/69","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"easy","template_id":"admin_016","template_name":"custom_obj_with_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/69"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/69"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/custom_obj_with_tab"}],"source_schema_notes":[],"template_registry":{"name":"custom_obj_with_tab","original":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nBuild a custom object named Bike, and have the New Custom Tab Wizard auto-launch on save by checking the “Launch New Custom Tab Wizard after saving this custom object” option. Choose any tab style you want and leave the rest as-is (defaults only).\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_004","query_templated":"Create a custom object with label Bike. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nBuild a custom object named Bike, and have the New Custom Tab Wizard auto-launch on save by checking the “Launch New Custom Tab Wizard after saving this custom object” option. Choose any tab style you want and leave the rest as-is (defaults only).\n```","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"Bike"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"Bike"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/69"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L274","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_custom_obj_with_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_tabs","params":{"types_and_members":{"CustomTab":["{custom_obj_name}__c"],"CustomObject":["{custom_obj_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"426f9eaaae0dfcd0db832fedc608b436a3f8f4e5132fdf23e530d475a9b36f34"} {"id":"scuba:admin_016_005","source_id":"scuba","original_id":"admin_016_005","kind":"benchmark_task","instruction":"I need a custom object labeled MyEnvironment. Ensure the “Launch New Custom Tab Wizard after saving this custom object” box is checked so I can pick any tab style, and use default values for everything else.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_005","query_templated":"Create a custom object with label MyEnvironment. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"I need a custom object labeled MyEnvironment. Ensure the “Launch New Custom Tab Wizard after saving this custom object” box is checked so I can pick any tab style, and use default values for everything else.","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"MyEnvironment"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"MyEnvironment"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/70","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"easy","template_id":"admin_016","template_name":"custom_obj_with_tab","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/70"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/70"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/custom_obj_with_tab"}],"source_schema_notes":[],"template_registry":{"name":"custom_obj_with_tab","original":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/custom_obj_with_tab","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nI need a custom object labeled MyEnvironment. Ensure the “Launch New Custom Tab Wizard after saving this custom object” box is checked so I can pick any tab style, and use default values for everything else.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_016_005","query_templated":"Create a custom object with label MyEnvironment. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish.\n\n## Your ultimate task\n```\nI need a custom object labeled MyEnvironment. Ensure the “Launch New Custom Tab Wizard after saving this custom object” box is checked so I can pick any tab style, and use default values for everything else.\n```","human_review_comment":"accept","ground_truth_dict":{"custom_obj_name":"MyEnvironment"},"difficulty":"easy","query_template_name":"custom_obj_with_tab","query_template_metadata":{"template_id":"admin_016","template_string":"Create a custom object with label {custom_obj_name}. Check \"Launch New Custom Tab Wizard after saving this custom object\" box to select the tab style of your choice. Use the default values for the remaining.","prerequisites":{},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"custom_obj_name":"MyEnvironment"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named [WildBird]. Check \"Launch New Custom Tab Wizard after saving this custom object\" box and select \"Tab Style: [Bridge]\" and finish the remaining steps with the default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and navigate to Setup → Object Manager.\n- Create → Custom Object.\n- Enter:\n - Label: WildBird\n - Plural Label: WildBirds\n- Check the box “Launch New Custom Tab Wizard after saving this custom object.”\n- Save.\n- In the New Custom Tab Wizard:\n - Click Tab Style and select “Bridge.”\n - Click Next, keeping default settings.\n - Click Next again, keeping default settings.\n - Click Save to finish.\n\nNotes:\n- Scrolling may be needed to locate the checkbox and action buttons.\n- No hot-keys were used.\nSample 2:\nTASK:Create a custom object named [StormTracker]. Check \"Launch New Custom Tab Wizard after saving this custom object,\" select Tab Style: [Lightning], and finish with default values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Navigate to Setup > Create > Custom Object.\n- Enter object details:\n - Label: StormTracker\n - Plural Label: StormTrackers\n - Ensure “Launch New Custom Tab Wizard after saving this custom object” is checked.\n- Click Save to launch the New Custom Tab Wizard.\n- In the Tab Style step, select “Lightning” and click Next.\n- Proceed through the remaining wizard pages with default values by clicking Next as needed.\n- Click Save to finish."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/70"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L274","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_custom_obj_with_tab","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_tabs","params":{"types_and_members":{"CustomTab":["{custom_obj_name}__c"],"CustomObject":["{custom_obj_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"32f7030cb19953d7794c20aad7036dda3f886fa638c5575b44c7bf1785a2bab1"} {"id":"scuba:admin_017_001","source_id":"scuba","original_id":"admin_017_001","kind":"benchmark_task","instruction":"Could you hop into the MyAnimal object and change its description to \"Updated Description\"? Also, update the Breed field’s help text so it says \"Updated Help text for breed\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_001","query_templated":"Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.","query":"Could you hop into the MyAnimal object and change its description to \"Updated Description\"? Also, update the Breed field’s help text so it says \"Updated Help text for breed\".","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyAnimal","description_text":"Updated Description","help_text":"Updated Help text for breed","field_name":"Breed"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyAnimal","description_text":"Updated Description","help_text":"Updated Help text for breed","field_name":"Breed"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/71","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_017","template_name":"update_custom_object_description_and_help_text","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/71"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/71"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/update_custom_object_description_and_help_text"}],"source_schema_notes":[],"template_registry":{"name":"update_custom_object_description_and_help_text","original":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nCould you hop into the MyAnimal object and change its description to \"Updated Description\"? Also, update the Breed field’s help text so it says \"Updated Help text for breed\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_001","query_templated":"Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nCould you hop into the MyAnimal object and change its description to \"Updated Description\"? Also, update the Breed field’s help text so it says \"Updated Help text for breed\".\n```","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyAnimal","description_text":"Updated Description","help_text":"Updated Help text for breed","field_name":"Breed"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyAnimal","description_text":"Updated Description","help_text":"Updated Help text for breed","field_name":"Breed"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/71"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L339","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_custom_object_description_and_help_text","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"CustomField":["{object_name}__c.{field_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e34b7e001b4e22e5b5ffc42c2cfc682d095a638ff88bcd0c6349fd0d1dc52724"} {"id":"scuba:admin_017_002","source_id":"scuba","original_id":"admin_017_002","kind":"benchmark_task","instruction":"For MyBike, make two edits: - Set the object description to \"This is the name of the Bike\" - For the Brand field, set the help text to \"This is the brand name of the Bike\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_002","query_templated":"Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.","query":"For MyBike, make two edits: - Set the object description to \"This is the name of the Bike\" - For the Brand field, set the help text to \"This is the brand name of the Bike\".","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyBike","description_text":"This is the name of the Bike","help_text":"This is the brand name of the Bike","field_name":"Brand"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyBike","description_text":"This is the name of the Bike","help_text":"This is the brand name of the Bike","field_name":"Brand"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/72","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_017","template_name":"update_custom_object_description_and_help_text","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/72"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/72"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/update_custom_object_description_and_help_text"}],"source_schema_notes":[],"template_registry":{"name":"update_custom_object_description_and_help_text","original":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nFor MyBike, make two edits: - Set the object description to \"This is the name of the Bike\" - For the Brand field, set the help text to \"This is the brand name of the Bike\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_002","query_templated":"Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nFor MyBike, make two edits: - Set the object description to \"This is the name of the Bike\" - For the Brand field, set the help text to \"This is the brand name of the Bike\".\n```","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyBike","description_text":"This is the name of the Bike","help_text":"This is the brand name of the Bike","field_name":"Brand"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyBike","description_text":"This is the name of the Bike","help_text":"This is the brand name of the Bike","field_name":"Brand"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/72"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L339","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_custom_object_description_and_help_text","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"CustomField":["{object_name}__c.{field_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6541d2ed6e6702b8d3bcc24c84f43acbfaced811e3e8b603bd66766dcf082da5"} {"id":"scuba:admin_017_003","source_id":"scuba","original_id":"admin_017_003","kind":"benchmark_task","instruction":"I’m tidying up metadata—please replace the Environment object’s description with \"This describes a new Salesforce Environment\" and update the Age field help text to \"Enter the number of days this env is live\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_003","query_templated":"Update the description of Environment object with the text This describes a new Salesforce Environment. And update the help text of the Age field to Enter the number of days this env is live.","query":"I’m tidying up metadata—please replace the Environment object’s description with \"This describes a new Salesforce Environment\" and update the Age field help text to \"Enter the number of days this env is live\"","human_review_comment":"delete .","ground_truth_dict":{"object_name":"Environment","description_text":"This describes a new Salesforce Environment","help_text":"Enter the number of days this env is live","field_name":"Age"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"Environment","description_text":"This describes a new Salesforce Environment","help_text":"Enter the number of days this env is live","field_name":"Age"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/73","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_017","template_name":"update_custom_object_description_and_help_text","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/73"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/73"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/update_custom_object_description_and_help_text"}],"source_schema_notes":[],"template_registry":{"name":"update_custom_object_description_and_help_text","original":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nI’m tidying up metadata—please replace the Environment object’s description with \"This describes a new Salesforce Environment\" and update the Age field help text to \"Enter the number of days this env is live\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_003","query_templated":"Update the description of Environment object with the text This describes a new Salesforce Environment. And update the help text of the Age field to Enter the number of days this env is live.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nI’m tidying up metadata—please replace the Environment object’s description with \"This describes a new Salesforce Environment\" and update the Age field help text to \"Enter the number of days this env is live\"\n```","human_review_comment":"delete .","ground_truth_dict":{"object_name":"Environment","description_text":"This describes a new Salesforce Environment","help_text":"Enter the number of days this env is live","field_name":"Age"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"Environment","description_text":"This describes a new Salesforce Environment","help_text":"Enter the number of days this env is live","field_name":"Age"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/73"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L339","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_custom_object_description_and_help_text","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"CustomField":["{object_name}__c.{field_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d7be5fa0115bcc86ecc3dcbe9b66d77073548336c1e7978aac4def6888dc6acc"} {"id":"scuba:admin_017_004","source_id":"scuba","original_id":"admin_017_004","kind":"benchmark_task","instruction":"Do the following on MyVehicle: 1) Set the description to \"Contains details about vehicle, like number of wheels, engine type, etc.\" 2) Update the NumWheels field help text to \"Describes the number of wheels in the vehicle\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_004","query_templated":"Update the description of MyVehicle object with the text Contains details about vehicle, like number of wheels, engine type, etc.. And update the help text of the NumWheels field to Describes the number of wheels in the vehicle.","query":"Do the following on MyVehicle: 1) Set the description to \"Contains details about vehicle, like number of wheels, engine type, etc.\" 2) Update the NumWheels field help text to \"Describes the number of wheels in the vehicle\".","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyVehicle","description_text":"Contains details about vehicle, like number of wheels, engine type, etc.","field_name":"NumWheels","help_text":"Describes the number of wheels in the vehicle"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyVehicle","description_text":"Contains details about vehicle, like number of wheels, engine type, etc.","field_name":"NumWheels","help_text":"Describes the number of wheels in the vehicle"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/74","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_017","template_name":"update_custom_object_description_and_help_text","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/74"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/74"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/update_custom_object_description_and_help_text"}],"source_schema_notes":[],"template_registry":{"name":"update_custom_object_description_and_help_text","original":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nDo the following on MyVehicle: 1) Set the description to \"Contains details about vehicle, like number of wheels, engine type, etc.\" 2) Update the NumWheels field help text to \"Describes the number of wheels in the vehicle\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_004","query_templated":"Update the description of MyVehicle object with the text Contains details about vehicle, like number of wheels, engine type, etc.. And update the help text of the NumWheels field to Describes the number of wheels in the vehicle.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nDo the following on MyVehicle: 1) Set the description to \"Contains details about vehicle, like number of wheels, engine type, etc.\" 2) Update the NumWheels field help text to \"Describes the number of wheels in the vehicle\".\n```","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyVehicle","description_text":"Contains details about vehicle, like number of wheels, engine type, etc.","field_name":"NumWheels","help_text":"Describes the number of wheels in the vehicle"},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyVehicle","description_text":"Contains details about vehicle, like number of wheels, engine type, etc.","field_name":"NumWheels","help_text":"Describes the number of wheels in the vehicle"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/74"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L339","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_custom_object_description_and_help_text","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"CustomField":["{object_name}__c.{field_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f913f089cf94f5ac4b6ef3d9630d963d1d44d412c733d446179e05e8d0d51997"} {"id":"scuba:admin_017_005","source_id":"scuba","original_id":"admin_017_005","kind":"benchmark_task","instruction":"While you’re at it, adjust MyInventory as follows: change the object description to \"Manages current inventory items, quantities, and stock levels.\" and set the StockLevel field’s help text to \"Specify how many units are currently available in stock.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_005","query_templated":"Update the description of MyInventory object with the text Manages current inventory items, quantities, and stock levels.. And update the help text of the StockLevel field to Specify how many units are currently available in stock..","query":"While you’re at it, adjust MyInventory as follows: change the object description to \"Manages current inventory items, quantities, and stock levels.\" and set the StockLevel field’s help text to \"Specify how many units are currently available in stock.\"","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyInventory","description_text":"Manages current inventory items, quantities, and stock levels.","field_name":"StockLevel","help_text":"Specify how many units are currently available in stock."},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyInventory","description_text":"Manages current inventory items, quantities, and stock levels.","field_name":"StockLevel","help_text":"Specify how many units are currently available in stock."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/75","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_017","template_name":"update_custom_object_description_and_help_text","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/75"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/75"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/update_custom_object_description_and_help_text"}],"source_schema_notes":[],"template_registry":{"name":"update_custom_object_description_and_help_text","original":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/update_custom_object_description_and_help_text","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nWhile you’re at it, adjust MyInventory as follows: change the object description to \"Manages current inventory items, quantities, and stock levels.\" and set the StockLevel field’s help text to \"Specify how many units are currently available in stock.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_017_005","query_templated":"Update the description of MyInventory object with the text Manages current inventory items, quantities, and stock levels.. And update the help text of the StockLevel field to Specify how many units are currently available in stock..","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save.\n\n## Your ultimate task\n```\nWhile you’re at it, adjust MyInventory as follows: change the object description to \"Manages current inventory items, quantities, and stock levels.\" and set the StockLevel field’s help text to \"Specify how many units are currently available in stock.\"\n```","human_review_comment":"delete .","ground_truth_dict":{"object_name":"MyInventory","description_text":"Manages current inventory items, quantities, and stock levels.","field_name":"StockLevel","help_text":"Specify how many units are currently available in stock."},"difficulty":"hard","query_template_name":"update_custom_object_description_and_help_text","query_template_metadata":{"template_id":"admin_017","template_string":"Update the description of {object_name} object with the text {description_text}. And update the help text of the {field_name} field to {help_text}.","prerequisites":{"metadata":{"types_and_members":{"CustomField":["MyAnimal__c.Breed__c","Environment__c.Age__c","MyBike__c.Brand__c","MyVehicle__c.NumWheels__c","MyInventory__c.StockLevel__c"],"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyVehicle__c","MyInventory__c"]}}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"object_name":"MyInventory","description_text":"Manages current inventory items, quantities, and stock levels.","field_name":"StockLevel","help_text":"Specify how many units are currently available in stock."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the description of MyAnimal object with the text Updated Description. And update the help text of the Breed field to Updated Help text for breed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup, use Quick Find to search for “Object Manager” and open it.\n- In Object Manager, use Quick Find to search for “MyAnimal” and open the MyAnimal object.\n- Click Edit for the MyAnimal object, update the Description field to: Updated Description, then Save.\n- Go to Fields & Relationships, open the Breed field, click Edit, update the Help Text to: Updated Help text for breed, then Save.\nSample 2:\nTASK:Update the description of MyBike object with the text This is the name of the Bike. And update the help text of the Brand field to This is the brand name of the Bike.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup, use Quick Find to search for “Object Manager,” and open Object Manager.\n- In Object Manager, search for “MyBike” and open the MyBike object.\n- Update Brand field help text:\n - Go to Fields & Relationships.\n - Open the Brand field and click Edit.\n - Set Help Text to: “This is the brand name of the Bike.”\n - Save.\n- Update object description:\n - Go to the Details tab of the MyBike object.\n - Click Edit.\n - Set Description to: “This is the name of the Bike.”\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/75"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L339","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_custom_object_description_and_help_text","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"CustomField":["{object_name}__c.{field_name}__c"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3becaf2f952eea141b87d650d0070ffc7eff2509a3efbccd902e30075fd46012"} {"id":"scuba:admin_018_001","source_id":"scuba","original_id":"admin_018_001","kind":"benchmark_task","instruction":"Go ahead and create a custom object called Favorite, hook it to Contact object with a Lookup Relationship, and lock down access so only the System Administrator and Solution Manager profiles can see it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_001","query_templated":"Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.","query":"Go ahead and create a custom object called Favorite, hook it to Contact object with a Lookup Relationship, and lock down access so only the System Administrator and Solution Manager profiles can see it.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Favorite","related_object":"Contact","comma_separated_profile_list":"Admin, SolutionManager"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Favorite","related_object":"Contact","comma_separated_profile_list":"System Administrator, Solution Manager"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/76","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_018","template_name":"create_custom_object_with_lookup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/76"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/76"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_object_with_lookup"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_object_with_lookup","original":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nGo ahead and create a custom object called Favorite, hook it to Contact object with a Lookup Relationship, and lock down access so only the System Administrator and Solution Manager profiles can see it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_001","query_templated":"Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nGo ahead and create a custom object called Favorite, hook it to Contact object with a Lookup Relationship, and lock down access so only the System Administrator and Solution Manager profiles can see it.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Favorite","related_object":"Contact","comma_separated_profile_list":"Admin, SolutionManager"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Favorite","related_object":"Contact","comma_separated_profile_list":"System Administrator, Solution Manager"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/76"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L233","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_object_with_lookup","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_profiles","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4ea2b6817e7b3e7b07c7d302f61a7c3506700a79fda20dd5028a7eddb7125d82"} {"id":"scuba:admin_018_002","source_id":"scuba","original_id":"admin_018_002","kind":"benchmark_task","instruction":"Build a custom object named ToughToClose. Add a Lookup relationship pointing to Opportunity. Limit object visibility to just the Customer Portal User and Standard User profiles.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_002","query_templated":"Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.","query":"Build a custom object named ToughToClose. Add a Lookup relationship pointing to Opportunity. Limit object visibility to just the Customer Portal User and Standard User profiles.","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/77","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_018","template_name":"create_custom_object_with_lookup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/77"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/77"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_object_with_lookup"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_object_with_lookup","original":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nBuild a custom object named ToughToClose. Add a Lookup relationship pointing to Opportunity. Limit object visibility to just the Customer Portal User and Standard User profiles.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_002","query_templated":"Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nBuild a custom object named ToughToClose. Add a Lookup relationship pointing to Opportunity. Limit object visibility to just the Customer Portal User and Standard User profiles.\n```","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/77"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L233","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_object_with_lookup","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_profiles","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"1e380c0b97ef34e1da0e3b95e604928accb729c9caaedbdc9272c4cbe2574f2d"} {"id":"scuba:admin_018_003","source_id":"scuba","original_id":"admin_018_003","kind":"benchmark_task","instruction":"We’re tracking hard-to-win deals—please create a custom object called ToughToClose, relate it to Opportunity using a Lookup, and make it viewable exclusively by the Customer Portal User and Standard User profiles.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_003","query_templated":"Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.","query":"We’re tracking hard-to-win deals—please create a custom object called ToughToClose, relate it to Opportunity using a Lookup, and make it viewable exclusively by the Customer Portal User and Standard User profiles.","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/78","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_018","template_name":"create_custom_object_with_lookup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/78"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/78"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_object_with_lookup"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_object_with_lookup","original":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nWe’re tracking hard-to-win deals—please create a custom object called ToughToClose, relate it to Opportunity using a Lookup, and make it viewable exclusively by the Customer Portal User and Standard User profiles.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_003","query_templated":"Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nWe’re tracking hard-to-win deals—please create a custom object called ToughToClose, relate it to Opportunity using a Lookup, and make it viewable exclusively by the Customer Portal User and Standard User profiles.\n```","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"ToughToClose","related_object":"Opportunity","comma_separated_profile_list":"Customer Portal User, Standard User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/78"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L233","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_object_with_lookup","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_profiles","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5508b6b114f57b2310eb7c736d7a460cffabd945dd41dec5370c4950dd559b33"} {"id":"scuba:admin_018_004","source_id":"scuba","original_id":"admin_018_004","kind":"benchmark_task","instruction":"Please provision a custom object titled Attention, attach a lookup relationship to Case object, and restrict availability so that only the Einstein Agent User and Standard Platform User profiles have access.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_004","query_templated":"Create a custom object named Attention. Then create a Lookup Relationship with Case object. It should only be visible to Einstein Agent User, Standard Platform User profiles.","query":"Please provision a custom object titled Attention, attach a lookup relationship to Case object, and restrict availability so that only the Einstein Agent User and Standard Platform User profiles have access.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Attention","related_object":"Case","comma_separated_profile_list":"Einstein Agent User, StandardAul"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Attention","related_object":"Case","comma_separated_profile_list":"Einstein Agent User, Standard Platform User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/79","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_018","template_name":"create_custom_object_with_lookup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/79"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/79"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_object_with_lookup"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_object_with_lookup","original":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nPlease provision a custom object titled Attention, attach a lookup relationship to Case object, and restrict availability so that only the Einstein Agent User and Standard Platform User profiles have access.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_004","query_templated":"Create a custom object named Attention. Then create a Lookup Relationship with Case object. It should only be visible to Einstein Agent User, Standard Platform User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nPlease provision a custom object titled Attention, attach a lookup relationship to Case object, and restrict availability so that only the Einstein Agent User and Standard Platform User profiles have access.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Attention","related_object":"Case","comma_separated_profile_list":"Einstein Agent User, StandardAul"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Attention","related_object":"Case","comma_separated_profile_list":"Einstein Agent User, Standard Platform User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/79"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L233","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_object_with_lookup","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_profiles","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f60598c894ad5321ec1868b9a7aed0d5fd526858f1cf5434d8c11807707438fb"} {"id":"scuba:admin_018_005","source_id":"scuba","original_id":"admin_018_005","kind":"benchmark_task","instruction":"I’d like a custom object named Throughput; tie it to Lead with a Lookup Relationship, and ensure it’s only visible to the Identity User and Contract Manager profiles.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_005","query_templated":"Create a custom object named Throughput. Then create a Lookup Relationship with Lead object. It should only be visible to Identity User, Contract Manager profiles.","query":"I’d like a custom object named Throughput; tie it to Lead with a Lookup Relationship, and ensure it’s only visible to the Identity User and Contract Manager profiles.","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"Throughput","related_object":"Lead","comma_separated_profile_list":"Identity User, ContractManager"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Throughput","related_object":"Lead","comma_separated_profile_list":"Identity User, Contract Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/80","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Data Model","source_difficulty":"hard","template_id":"admin_018","template_name":"create_custom_object_with_lookup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/80"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/80"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_custom_object_with_lookup"}],"source_schema_notes":[],"template_registry":{"name":"create_custom_object_with_lookup","original":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_custom_object_with_lookup","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nI’d like a custom object named Throughput; tie it to Lead with a Lookup Relationship, and ensure it’s only visible to the Identity User and Contract Manager profiles.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Data Model","task_id":"admin_018_005","query_templated":"Create a custom object named Throughput. Then create a Lookup Relationship with Lead object. It should only be visible to Identity User, Contract Manager profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles.\n\n## Your ultimate task\n```\nI’d like a custom object named Throughput; tie it to Lead with a Lookup Relationship, and ensure it’s only visible to the Identity User and Contract Manager profiles.\n```","human_review_comment":"rewrite lookup","ground_truth_dict":{"object_name":"Throughput","related_object":"Lead","comma_separated_profile_list":"Identity User, ContractManager"},"difficulty":"hard","query_template_name":"create_custom_object_with_lookup","query_template_metadata":{"template_id":"admin_018","template_string":"Create a custom object named {object_name}. Then create a Lookup Relationship with {related_object} object. It should only be visible to {comma_separated_profile_list} profiles.","prerequisites":{"data":{"objects":["Profile"]}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Throughput","related_object":"Lead","comma_separated_profile_list":"Identity User, Contract Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a custom object named Favorite. Then create a Lookup Relationship with Contact object. It should only be visible to System Administrator, Solution Manager profiles.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Create > Custom Object.\n - Create a new custom object with:\n - Label: Favorite\n - Plural Label: Favorites\n - Save.\n- Open the Favorite object, go to Fields & Relationships, and click New.\n - Choose data type: Lookup Relationship, then Next.\n - Related To: Contact, then Next.\n - Accept defaults for field name, then Next as needed.\n - Field-Level Security: ensure Visible is checked only for the System Administrator and Solution Manager profiles (uncheck all other profiles).\n - Continue through the remaining Next steps (page layout/related list) and Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Create a custom object named ToughToClose. Then create a Lookup Relationship with Opportunity object. It should only be visible to Customer Portal User, Standard User profiles.\nEXPERIENCE:\nSuccessful plan to create ToughToClose object and a lookup to Opportunity with restricted visibility:\n\n1) Access Setup\n- Log in.\n- Click Setup.\n\n2) Create the custom object\n- Go to Create > Custom Object.\n- Enter Label: ToughToClose; Plural Label: ToughToCloses.\n- Click Save.\n\n3) Add a Lookup Relationship to Opportunity\n- In the ToughToClose object, open Fields & Relationships.\n- Click New.\n- Select Lookup Relationship and click Next.\n- Set Related To = Opportunity and click Next.\n- Accept defaults on subsequent step(s) and click Next.\n\n4) Set field-level security (visibility)\n- On the Field-Level Security step, clear default visibility if needed.\n- Check Visible only for:\n - Customer Portal User\n - Standard User\n- Click Next.\n\n5) Finalize\n- Accept defaults for page layout assignment (Next).\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Visibility was explicitly set only for Customer Portal User and Standard User profiles."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/80"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L233","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_custom_object_with_lookup","data_retrieval_spec":[{"type":"metadata","id":"custom_objects_and_profiles","params":{"types_and_members":{"CustomObject":["{object_name}__c"],"Profile":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2eae49147d8f58d738f6a62b858227bb7b1c58387750cb8a65b372dae5e33a3c"} {"id":"scuba:admin_019_001","source_id":"scuba","original_id":"admin_019_001","kind":"benchmark_task","instruction":"Please set up a new Permission Set named “Account Access.” Leave the license as None. On the Account object, grant object permissions for Read and Create. Also give Read and Edit access to all fields.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_001","query_templated":"Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.","query":"Please set up a new Permission Set named “Account Access.” Leave the license as None. On the Account object, grant object permissions for Read and Create. Also give Read and Edit access to all fields.","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Account Access","object_name":"Account","permissions":"allowRead, allowCreate"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Account Access","object_name":"Account","permission1":"Read","permission2":"Create"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/81","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_019","template_name":"create_permission_set_medium","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/81"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/81"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permission_set_medium"}],"source_schema_notes":[],"template_registry":{"name":"create_permission_set_medium","original":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease set up a new Permission Set named “Account Access.” Leave the license as None. On the Account object, grant object permissions for Read and Create. Also give Read and Edit access to all fields.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_001","query_templated":"Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease set up a new Permission Set named “Account Access.” Leave the license as None. On the Account object, grant object permissions for Read and Create. Also give Read and Edit access to all fields.\n```","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Account Access","object_name":"Account","permissions":"allowRead, allowCreate"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Account Access","object_name":"Account","permission1":"Read","permission2":"Create"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/81"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L108","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permission_set_medium","data_retrieval_spec":[{"type":"metadata","id":"permission_sets","params":{"types_and_members":{"PermissionSet":["{'_'.join(label_name.split(' '))}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e1c93233e5f441c2b4ba2a065079f125e0803ed3bc560addb30fb7215526a7ef"} {"id":"scuba:admin_019_002","source_id":"scuba","original_id":"admin_019_002","kind":"benchmark_task","instruction":"I need a Permission Set with these specs: - Label: Opportunity Access - License: None - Opportunity Object Permissions: Read, Edit. Read and Edit for every field","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_002","query_templated":"Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunity Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.","query":"I need a Permission Set with these specs: - Label: Opportunity Access - License: None - Opportunity Object Permissions: Read, Edit. Read and Edit for every field","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Opportunity Access","object_name":"Opportunity","permissions":"allowRead, allowEdit"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Opportunity Access","object_name":"Opportunity","permission1":"Read","permission2":"Edit"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/82","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_019","template_name":"create_permission_set_medium","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/82"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/82"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permission_set_medium"}],"source_schema_notes":[],"template_registry":{"name":"create_permission_set_medium","original":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI need a Permission Set with these specs: - Label: Opportunity Access - License: None - Opportunity Object Permissions: Read, Edit. Read and Edit for every field\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_002","query_templated":"Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunity Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI need a Permission Set with these specs: - Label: Opportunity Access - License: None - Opportunity Object Permissions: Read, Edit. Read and Edit for every field\n```","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Opportunity Access","object_name":"Opportunity","permissions":"allowRead, allowEdit"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Opportunity Access","object_name":"Opportunity","permission1":"Read","permission2":"Edit"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/82"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L108","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permission_set_medium","data_retrieval_spec":[{"type":"metadata","id":"permission_sets","params":{"types_and_members":{"PermissionSet":["{'_'.join(label_name.split(' '))}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5879ff3c2ce5c404ddb6f6afc328a034d18bcd0556a84d69fe65a31fcf3fa04a"} {"id":"scuba:admin_019_003","source_id":"scuba","original_id":"admin_019_003","kind":"benchmark_task","instruction":"Create a Permission Set and call it “Address Access” (license should be None). For the Address object, turn on Read, Edit, Delete, and View All Records. Then make sure every field is set to Read and Edit.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_003","query_templated":"Create a Permission set by setting the label as Address Access. The license is set to None. Enable Address Object Permissions: Read, Edit, Delete, View All Records. Enable Read and Edit Access for all fields.","query":"Create a Permission Set and call it “Address Access” (license should be None). For the Address object, turn on Read, Edit, Delete, and View All Records. Then make sure every field is set to Read and Edit.","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Address Access","object_name":"Address","permissions":"allowRead, allowEdit, allowDelete, viewAllRecords"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Address Access","object_name":"Address","permission1":"Read, Edit, Delete","permission2":"View All Records"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/83","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_019","template_name":"create_permission_set_medium","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/83"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/83"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permission_set_medium"}],"source_schema_notes":[],"template_registry":{"name":"create_permission_set_medium","original":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate a Permission Set and call it “Address Access” (license should be None). For the Address object, turn on Read, Edit, Delete, and View All Records. Then make sure every field is set to Read and Edit.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_003","query_templated":"Create a Permission set by setting the label as Address Access. The license is set to None. Enable Address Object Permissions: Read, Edit, Delete, View All Records. Enable Read and Edit Access for all fields.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate a Permission Set and call it “Address Access” (license should be None). For the Address object, turn on Read, Edit, Delete, and View All Records. Then make sure every field is set to Read and Edit.\n```","human_review_comment":"rewrite","ground_truth_dict":{"label_name":"Address Access","object_name":"Address","permissions":"allowRead, allowEdit, allowDelete, viewAllRecords"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Address Access","object_name":"Address","permission1":"Read, Edit, Delete","permission2":"View All Records"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/83"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L108","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permission_set_medium","data_retrieval_spec":[{"type":"metadata","id":"permission_sets","params":{"types_and_members":{"PermissionSet":["{'_'.join(label_name.split(' '))}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c713c6dca4886c66ca0ed26ab5fcbc605a0e6fb7b18201ecb618f3dc8c2162d7"} {"id":"scuba:admin_019_004","source_id":"scuba","original_id":"admin_019_004","kind":"benchmark_task","instruction":"Set up a Permission Set labeled “Actions Access,” with the license left as None. Configure the Shipment object so it has Read, Edit, and Delete object permissions. Ensure all fields have both Read and Edit enabled.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_004","query_templated":"Create a Permission set by setting the label as Actions Access. The license is set to None. Enable Shipment Object Permissions: Read, Edit, Delete. Enable Read and Edit Access for all fields.","query":"Set up a Permission Set labeled “Actions Access,” with the license left as None. Configure the Shipment object so it has Read, Edit, and Delete object permissions. Ensure all fields have both Read and Edit enabled.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Actions Access","object_name":"Shipment","permissions":"allowRead, allowEdit, allowDelete"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Actions Access","object_name":"Shipment","permission1":"Read, Edit","permission2":"Delete"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/84","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_019","template_name":"create_permission_set_medium","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/84"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/84"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permission_set_medium"}],"source_schema_notes":[],"template_registry":{"name":"create_permission_set_medium","original":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nSet up a Permission Set labeled “Actions Access,” with the license left as None. Configure the Shipment object so it has Read, Edit, and Delete object permissions. Ensure all fields have both Read and Edit enabled.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_004","query_templated":"Create a Permission set by setting the label as Actions Access. The license is set to None. Enable Shipment Object Permissions: Read, Edit, Delete. Enable Read and Edit Access for all fields.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nSet up a Permission Set labeled “Actions Access,” with the license left as None. Configure the Shipment object so it has Read, Edit, and Delete object permissions. Ensure all fields have both Read and Edit enabled.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Actions Access","object_name":"Shipment","permissions":"allowRead, allowEdit, allowDelete"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Actions Access","object_name":"Shipment","permission1":"Read, Edit","permission2":"Delete"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/84"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L108","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permission_set_medium","data_retrieval_spec":[{"type":"metadata","id":"permission_sets","params":{"types_and_members":{"PermissionSet":["{'_'.join(label_name.split(' '))}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f21f88cd6bf5ceec415a6ee7b86cc3484c773ccfdb4dd9841a523dd76ebe578b"} {"id":"scuba:admin_019_005","source_id":"scuba","original_id":"admin_019_005","kind":"benchmark_task","instruction":"Please build a Permission Set called “Cases Admin.” Keep the license set to None. On the Case object, allow Read and Delete. Grant Read and Edit access across all fields.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_005","query_templated":"Create a Permission set by setting the label as Cases Admin. The license is set to None. Enable Case Object Permissions: Read, Delete. Enable Read and Edit Access for all fields.","query":"Please build a Permission Set called “Cases Admin.” Keep the license set to None. On the Case object, allow Read and Delete. Grant Read and Edit access across all fields.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Cases Admin","object_name":"Case","permissions":"allowRead, allowDelete, allowEdit"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Cases Admin","object_name":"Case","permission1":"Read","permission2":"Delete"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/85","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_019","template_name":"create_permission_set_medium","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/85"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/85"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permission_set_medium"}],"source_schema_notes":[],"template_registry":{"name":"create_permission_set_medium","original":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permission_set_medium","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease build a Permission Set called “Cases Admin.” Keep the license set to None. On the Case object, allow Read and Delete. Grant Read and Edit access across all fields.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_019_005","query_templated":"Create a Permission set by setting the label as Cases Admin. The license is set to None. Enable Case Object Permissions: Read, Delete. Enable Read and Edit Access for all fields.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease build a Permission Set called “Cases Admin.” Keep the license set to None. On the Case object, allow Read and Delete. Grant Read and Edit access across all fields.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Cases Admin","object_name":"Case","permissions":"allowRead, allowDelete, allowEdit"},"difficulty":"medium","query_template_name":"create_permission_set_medium","query_template_metadata":{"template_id":"admin_019","template_string":"Create a Permission set by setting the label as {label_name}. The license is set to None. Enable {object_name} Object Permissions: {permission1}, {permission2}. Enable Read and Edit Access for all fields.","prerequisites":{},"objects":["PermissionSet"],"metadata_types":["PermissionSet"]},"instance_dict":{"label_name":"Cases Admin","object_name":"Case","permission1":"Read","permission2":"Delete"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission set by setting the label as Account Access. The license is set to None. Enable Account Object Permissions: Read, Create. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “Permission Sets” and open it.\n- Click New to create a permission set.\n- Enter Label: “Account Access”. Click the API Name field to auto-populate it.\n- Ensure License is set to None (leave the default None).\n- Click Save.\n- In the permission set, go to Object Settings > Accounts > Edit.\n- Under Object Permissions, enable Read and Create.\n- Under Field Permissions, enable Read Access for all fields and Edit Access for all fields (use the select-all checkboxes).\n- Click Save.\nSample 2:\nTASK:Create a Permission set by setting the label as Opportunity Access. The license is set to None. Enable Opportunities Object Permissions: Read, Edit. Enable Read and Edit Access for all fields.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, search for “Permission Sets” and open it.\n- Click New to create a permission set.\n - Set Label to “Opportunity Access”. Click the API Name field to auto-populate it.\n - Leave License set to None.\n - Save.\n- In the created permission set, go to Object Settings.\n - Open Opportunities, click Edit.\n - Enable Object Permissions: check Read and Edit.\n - Enable field access: check “Read Access” and “Edit Access” for all fields.\n - Save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/85"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L108","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permission_set_medium","data_retrieval_spec":[{"type":"metadata","id":"permission_sets","params":{"types_and_members":{"PermissionSet":["{'_'.join(label_name.split(' '))}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c07b6d4bda521cbe925d61c6a0281aba0a9a14320f055bb5dcef5c826e7ea924"} {"id":"scuba:admin_020_001","source_id":"scuba","original_id":"admin_020_001","kind":"benchmark_task","instruction":"Create a Permission Set Group, give it the label Product Admin, and add the Merchandiser access set to it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_001","query_templated":"Create a Permission Set Group with the label name Product Admin. Include the Merchandiser access set.","query":"Create a Permission Set Group, give it the label Product Admin, and add the Merchandiser access set to it.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Product Admin","access_set_name":"Merchandiser"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Product Admin","access_set_name":"Merchandiser"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/86","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_020","template_name":"create_permset_group_access_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/86"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/86"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_access_set"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_access_set","original":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nCreate a Permission Set Group, give it the label Product Admin, and add the Merchandiser access set to it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_001","query_templated":"Create a Permission Set Group with the label name Product Admin. Include the Merchandiser access set.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nCreate a Permission Set Group, give it the label Product Admin, and add the Merchandiser access set to it.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Product Admin","access_set_name":"Merchandiser"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Product Admin","access_set_name":"Merchandiser"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/86"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_access_set","data_retrieval_spec":[{"type":"soql","id":"permission_set_group","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{label_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{label_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a39dd386d86847d751c2848b67465f646a88e309a331128fad8a1bb0b1081c58"} {"id":"scuba:admin_020_002","source_id":"scuba","original_id":"admin_020_002","kind":"benchmark_task","instruction":"When you get a moment, set up a Permission Set Group labeled Agentforce Data Admin1 and include the Data Cloud Admin access set.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_002","query_templated":"Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.","query":"When you get a moment, set up a Permission Set Group labeled Agentforce Data Admin1 and include the Data Cloud Admin access set.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Agentforce Data Admin1","access_set_name":"Data Cloud Admin"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Agentforce Data Admin1","access_set_name":"Data Cloud Admin"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/87","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_020","template_name":"create_permset_group_access_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/87"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/87"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_access_set"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_access_set","original":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nWhen you get a moment, set up a Permission Set Group labeled Agentforce Data Admin1 and include the Data Cloud Admin access set.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_002","query_templated":"Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nWhen you get a moment, set up a Permission Set Group labeled Agentforce Data Admin1 and include the Data Cloud Admin access set.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Agentforce Data Admin1","access_set_name":"Data Cloud Admin"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Agentforce Data Admin1","access_set_name":"Data Cloud Admin"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/87"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_access_set","data_retrieval_spec":[{"type":"soql","id":"permission_set_group","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{label_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{label_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e45397376f1019dbe59c21867a720304188935a0f3bed28adedfcb62dd528a40"} {"id":"scuba:admin_020_003","source_id":"scuba","original_id":"admin_020_003","kind":"benchmark_task","instruction":"Please provision a new Permission Set Group titled New Sales User and make sure the Salesforce Pricing Manager access set is part of it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_003","query_templated":"Create a Permission Set Group with the label name New Sales User. Include the Salesforce Pricing Manager access set.","query":"Please provision a new Permission Set Group titled New Sales User and make sure the Salesforce Pricing Manager access set is part of it.","human_review_comment":"accept","ground_truth_dict":{"label_name":"New Sales User","access_set_name":"Salesforce Pricing Manager"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"New Sales User","access_set_name":"Salesforce Pricing Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/88","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_020","template_name":"create_permset_group_access_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/88"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/88"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_access_set"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_access_set","original":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nPlease provision a new Permission Set Group titled New Sales User and make sure the Salesforce Pricing Manager access set is part of it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_003","query_templated":"Create a Permission Set Group with the label name New Sales User. Include the Salesforce Pricing Manager access set.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nPlease provision a new Permission Set Group titled New Sales User and make sure the Salesforce Pricing Manager access set is part of it.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"New Sales User","access_set_name":"Salesforce Pricing Manager"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"New Sales User","access_set_name":"Salesforce Pricing Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/88"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_access_set","data_retrieval_spec":[{"type":"soql","id":"permission_set_group","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{label_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{label_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c9ff83e10a37d2868a617329181f95703394644fd3c4224d5193313948f0674f"} {"id":"scuba:admin_020_004","source_id":"scuba","original_id":"admin_020_004","kind":"benchmark_task","instruction":"Set up a Permission Set Group with the label Survey Admin, then attach the Manage Assessment Surveys access set to that group.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_004","query_templated":"Create a Permission Set Group with the label name Survey Admin. Include the Manage Assessment Surveys access set.","query":"Set up a Permission Set Group with the label Survey Admin, then attach the Manage Assessment Surveys access set to that group.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Survey Admin","access_set_name":"Manage Assessment Surveys"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Survey Admin","access_set_name":"Manage Assessment Surveys"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/89","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_020","template_name":"create_permset_group_access_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/89"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/89"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_access_set"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_access_set","original":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nSet up a Permission Set Group with the label Survey Admin, then attach the Manage Assessment Surveys access set to that group.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_004","query_templated":"Create a Permission Set Group with the label name Survey Admin. Include the Manage Assessment Surveys access set.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nSet up a Permission Set Group with the label Survey Admin, then attach the Manage Assessment Surveys access set to that group.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Survey Admin","access_set_name":"Manage Assessment Surveys"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Survey Admin","access_set_name":"Manage Assessment Surveys"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/89"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_access_set","data_retrieval_spec":[{"type":"soql","id":"permission_set_group","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{label_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{label_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"151cb4f9ca2256c8b9041f1b04f0d26b4f51395ecd08bddd3d3e62c743c04d60"} {"id":"scuba:admin_020_005","source_id":"scuba","original_id":"admin_020_005","kind":"benchmark_task","instruction":"Build a Permission Set Group named Trainee Analyst and include the Use Enablement Programs access set in it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_005","query_templated":"Create a Permission Set Group with the label name Trainee Analyst. Include the Use Enablement Programs access set.","query":"Build a Permission Set Group named Trainee Analyst and include the Use Enablement Programs access set in it.","human_review_comment":"accept","ground_truth_dict":{"label_name":"Trainee Analyst","access_set_name":"Use Enablement Programs"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Trainee Analyst","access_set_name":"Use Enablement Programs"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/90","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_020","template_name":"create_permset_group_access_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/90"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/90"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_access_set"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_access_set","original":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_access_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nBuild a Permission Set Group named Trainee Analyst and include the Use Enablement Programs access set in it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_020_005","query_templated":"Create a Permission Set Group with the label name Trainee Analyst. Include the Use Enablement Programs access set.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”\n\n## Your ultimate task\n```\nBuild a Permission Set Group named Trainee Analyst and include the Use Enablement Programs access set in it.\n```","human_review_comment":"accept","ground_truth_dict":{"label_name":"Trainee Analyst","access_set_name":"Use Enablement Programs"},"difficulty":"easy","query_template_name":"create_permset_group_access_set","query_template_metadata":{"template_id":"admin_020","template_string":"Create a Permission Set Group with the label name {label_name}. Include the {access_set_name} access set.","prerequisites":{},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"label_name":"Trainee Analyst","access_set_name":"Use Enablement Programs"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Permission Set Group with the label name Product Admin. Include the Merchandiser permission set.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type \"Permission Set Groups\" and select it.\n- Click \"New Permission Set Group\".\n- Enter Label: \"Product Admin\" and click \"Save\".\n- In the \"Permission Sets in Group\" related list, click \"Add Permission Set\".\n- Filter to M (if needed), check \"Merchandiser\".\n- Click \"Add\", then \"Done\".\nSample 2:\nTASK:Create a Permission Set Group with the label name Agentforce Data Admin1. Include the Data Cloud Admin access set.\nEXPERIENCE:\nSuccessful plan to create the Permission Set Group and include Data Cloud Admin:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Permission Set Groups” and open it.\n- Click “New Permission Set Group.”\n- Enter the label: “Agentforce Data Admin1” and Save.\n- Open the “Permission Sets in Group” section and click “Add Permission Set.”\n- Use the index filter for the letter “D,” select the checkbox for “Data Cloud Admin,” click “Add,” then “Done.”"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/90"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_access_set","data_retrieval_spec":[{"type":"soql","id":"permission_set_group","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{label_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{label_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"589608df41a6597b92faf7feac60b00184804cfa1638f7ea79ea4f5f2bb9d744"} {"id":"scuba:admin_021_001","source_id":"scuba","original_id":"admin_021_001","kind":"benchmark_task","instruction":"Jump into the MarketingAnalyticsUser permission set group and mute the Accounts object permissions for Delete and Edit. Also mute Edit access on the Annual Revenue field(s).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_001","query_templated":"In the 'MarketingAnalyticsUser' permission set group, mute 'Accounts' object Permission on 'Delete' and 'Edit'. Mute the 'Edit Access' for the 'Annual Revenue' fields.","query":"Jump into the MarketingAnalyticsUser permission set group and mute the Accounts object permissions for Delete and Edit. Also mute Edit access on the Annual Revenue field(s).","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","object":"Account","object_permissions":"allowEdit, allowDelete, modifyAllRecords","field_permission_type1":"editable","field":"AnnualRevenue"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","object":"Accounts","object_permission_type1":"Delete","object_permission_type2":"Edit","field_permission_type1":"Edit Access","field":"Annual Revenue"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/91","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"hard","template_id":"admin_021","template_name":"create_muting_permission_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/91"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/91"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_muting_permission_set"}],"source_schema_notes":[],"template_registry":{"name":"create_muting_permission_set","original":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nJump into the MarketingAnalyticsUser permission set group and mute the Accounts object permissions for Delete and Edit. Also mute Edit access on the Annual Revenue field(s).\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_001","query_templated":"In the 'MarketingAnalyticsUser' permission set group, mute 'Accounts' object Permission on 'Delete' and 'Edit'. Mute the 'Edit Access' for the 'Annual Revenue' fields.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nJump into the MarketingAnalyticsUser permission set group and mute the Accounts object permissions for Delete and Edit. Also mute Edit access on the Annual Revenue field(s).\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","object":"Account","object_permissions":"allowEdit, allowDelete, modifyAllRecords","field_permission_type1":"editable","field":"AnnualRevenue"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","object":"Accounts","object_permission_type1":"Delete","object_permission_type2":"Edit","field_permission_type1":"Edit Access","field":"Annual Revenue"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/91"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L708","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_muting_permission_set","data_retrieval_spec":[{"type":"metadata","id":"permission_set_group","params":{"types_and_members":{"PermissionSetGroup":["{permission_set_group.replace(' ', '_')}"]}}},{"type":"metadata","id":"muting_permission_set","params":{"types_and_members":{"MutingPermissionSet":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"481c1f131dcb512897036f6b189e5fd75ccfdc5927e9c70955630d064904200c"} {"id":"scuba:admin_021_002","source_id":"scuba","original_id":"admin_021_002","kind":"benchmark_task","instruction":"In the Sales User permission set group, please mute the Addresses object by turning off Edit and Modify All Records, and also mute Read access for the AddressType field(s).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_002","query_templated":"In the 'Sales User' permission set group, mute 'Addresses' object Permission on 'Edit' and 'Modify All records'. Mute the 'Read Access' for the 'AddressType' fields.","query":"In the Sales User permission set group, please mute the Addresses object by turning off Edit and Modify All Records, and also mute Read access for the AddressType field(s).","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"Sales_User","object":"Address","object_permissions":"allowEdit, modifyAllRecords, allowDelete","field_permission_type1":"readable","field":"AddressType"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"Sales User","object":"Addresses","object_permission_type1":"Edit","object_permission_type2":"Modify All records","field_permission_type1":"Read Access","field":"AddressType"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/92","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"hard","template_id":"admin_021","template_name":"create_muting_permission_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/92"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/92"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_muting_permission_set"}],"source_schema_notes":[],"template_registry":{"name":"create_muting_permission_set","original":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the Sales User permission set group, please mute the Addresses object by turning off Edit and Modify All Records, and also mute Read access for the AddressType field(s).\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_002","query_templated":"In the 'Sales User' permission set group, mute 'Addresses' object Permission on 'Edit' and 'Modify All records'. Mute the 'Read Access' for the 'AddressType' fields.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the Sales User permission set group, please mute the Addresses object by turning off Edit and Modify All Records, and also mute Read access for the AddressType field(s).\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"Sales_User","object":"Address","object_permissions":"allowEdit, modifyAllRecords, allowDelete","field_permission_type1":"readable","field":"AddressType"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"Sales User","object":"Addresses","object_permission_type1":"Edit","object_permission_type2":"Modify All records","field_permission_type1":"Read Access","field":"AddressType"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/92"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L708","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_muting_permission_set","data_retrieval_spec":[{"type":"metadata","id":"permission_set_group","params":{"types_and_members":{"PermissionSetGroup":["{permission_set_group.replace(' ', '_')}"]}}},{"type":"metadata","id":"muting_permission_set","params":{"types_and_members":{"MutingPermissionSet":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"bdb078c9a1ff50dd96a59fcf81eb1efc8823fbf889bdce54f5a2ea2bda7ea14a"} {"id":"scuba:admin_021_003","source_id":"scuba","original_id":"admin_021_003","kind":"benchmark_task","instruction":"Tighten permissions in SamplePermsetGroup: mute Opportunities for Modify All Records and View All Fields, and mute Edit access on the Amount field(s).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_003","query_templated":"In the 'SamplePermsetGroup' permission set group, mute 'Opportunities' object Permission on 'Modify All Records' and 'View All Fields'. Mute the 'Edit Access' for the 'Amount' fields.","query":"Tighten permissions in SamplePermsetGroup: mute Opportunities for Modify All Records and View All Fields, and mute Edit access on the Amount field(s).","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"SamplePermsetGroup","object":"Opportunity","object_permissions":"viewAllFields, modifyAllRecords","field_permission_type1":"editable","field":"Amount"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"SamplePermsetGroup","object":"Opportunities","object_permission_type1":"Modify All Records","object_permission_type2":"View All Fields","field_permission_type1":"Edit Access","field":"Amount"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/93","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"hard","template_id":"admin_021","template_name":"create_muting_permission_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/93"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/93"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_muting_permission_set"}],"source_schema_notes":[],"template_registry":{"name":"create_muting_permission_set","original":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nTighten permissions in SamplePermsetGroup: mute Opportunities for Modify All Records and View All Fields, and mute Edit access on the Amount field(s).\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_003","query_templated":"In the 'SamplePermsetGroup' permission set group, mute 'Opportunities' object Permission on 'Modify All Records' and 'View All Fields'. Mute the 'Edit Access' for the 'Amount' fields.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nTighten permissions in SamplePermsetGroup: mute Opportunities for Modify All Records and View All Fields, and mute Edit access on the Amount field(s).\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"SamplePermsetGroup","object":"Opportunity","object_permissions":"viewAllFields, modifyAllRecords","field_permission_type1":"editable","field":"Amount"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"SamplePermsetGroup","object":"Opportunities","object_permission_type1":"Modify All Records","object_permission_type2":"View All Fields","field_permission_type1":"Edit Access","field":"Amount"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/93"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L708","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_muting_permission_set","data_retrieval_spec":[{"type":"metadata","id":"permission_set_group","params":{"types_and_members":{"PermissionSetGroup":["{permission_set_group.replace(' ', '_')}"]}}},{"type":"metadata","id":"muting_permission_set","params":{"types_and_members":{"MutingPermissionSet":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b62ddd30755a945023e09d8f6ebbcfc729fdb9440b75af5f5755c3c3641311b1"} {"id":"scuba:admin_021_004","source_id":"scuba","original_id":"admin_021_004","kind":"benchmark_task","instruction":"For MyNewPermissionSetGroup, mute Cases permissions for Create and View All Fields, then mute Read access on the Priority field(s).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_004","query_templated":"In the 'MyNewPermissionSetGroup' permission set group, mute 'Cases' object Permission on 'Create' and 'View All Fields'. Mute the 'Read Access' for the 'Priority' fields.","query":"For MyNewPermissionSetGroup, mute Cases permissions for Create and View All Fields, then mute Read access on the Priority field(s).","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MyNewPermissionSetGroup","object":"Case","object_permissions":"viewAllFields, allowCreate","field_permission_type1":"readable, editable","field":"Priority"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MyNewPermissionSetGroup","object":"Cases","object_permission_type1":"Create","object_permission_type2":"View All Fields","field_permission_type1":"Read Access","field":"Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/94","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"hard","template_id":"admin_021","template_name":"create_muting_permission_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/94"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/94"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_muting_permission_set"}],"source_schema_notes":[],"template_registry":{"name":"create_muting_permission_set","original":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFor MyNewPermissionSetGroup, mute Cases permissions for Create and View All Fields, then mute Read access on the Priority field(s).\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_004","query_templated":"In the 'MyNewPermissionSetGroup' permission set group, mute 'Cases' object Permission on 'Create' and 'View All Fields'. Mute the 'Read Access' for the 'Priority' fields.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFor MyNewPermissionSetGroup, mute Cases permissions for Create and View All Fields, then mute Read access on the Priority field(s).\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MyNewPermissionSetGroup","object":"Case","object_permissions":"viewAllFields, allowCreate","field_permission_type1":"readable, editable","field":"Priority"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MyNewPermissionSetGroup","object":"Cases","object_permission_type1":"Create","object_permission_type2":"View All Fields","field_permission_type1":"Read Access","field":"Priority"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/94"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L708","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_muting_permission_set","data_retrieval_spec":[{"type":"metadata","id":"permission_set_group","params":{"types_and_members":{"PermissionSetGroup":["{permission_set_group.replace(' ', '_')}"]}}},{"type":"metadata","id":"muting_permission_set","params":{"types_and_members":{"MutingPermissionSet":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c4b23dbbc961f3b74f22f48c0f4e6d9d309910f47ddd71798aa8b8de2a490836"} {"id":"scuba:admin_021_005","source_id":"scuba","original_id":"admin_021_005","kind":"benchmark_task","instruction":"In the LaLaLandUsers permission set group, mute the Shipment object permissions for Create and View All Fields, and mute Edit access for the ExpectedDeliveryDate field(s).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_005","query_templated":"In the 'LaLaLandUsers' permission set group, mute 'Shipment' object Permission on 'Create' and 'View All Fields'. Mute the 'Edit Access' for the 'ExpectedDeliveryDate' fields.","query":"In the LaLaLandUsers permission set group, mute the Shipment object permissions for Create and View All Fields, and mute Edit access for the ExpectedDeliveryDate field(s).","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"LaLaLandUsers","object":"Shipment","object_permissions":"viewAllFields, allowCreate","field_permission_type1":"editable","field":"ExpectedDeliveryDate"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"LaLaLandUsers","object":"Shipment","object_permission_type1":"Create","object_permission_type2":"View All Fields","field_permission_type1":"Edit Access","field":"ExpectedDeliveryDate"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/95","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"hard","template_id":"admin_021","template_name":"create_muting_permission_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/95"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/95"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_muting_permission_set"}],"source_schema_notes":[],"template_registry":{"name":"create_muting_permission_set","original":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_muting_permission_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the LaLaLandUsers permission set group, mute the Shipment object permissions for Create and View All Fields, and mute Edit access for the ExpectedDeliveryDate field(s).\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_021_005","query_templated":"In the 'LaLaLandUsers' permission set group, mute 'Shipment' object Permission on 'Create' and 'View All Fields'. Mute the 'Edit Access' for the 'ExpectedDeliveryDate' fields.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the LaLaLandUsers permission set group, mute the Shipment object permissions for Create and View All Fields, and mute Edit access for the ExpectedDeliveryDate field(s).\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"LaLaLandUsers","object":"Shipment","object_permissions":"viewAllFields, allowCreate","field_permission_type1":"editable","field":"ExpectedDeliveryDate"},"difficulty":"hard","query_template_name":"create_muting_permission_set","query_template_metadata":{"template_id":"admin_021","template_string":"In the '{permission_set_group}' permission set group, mute '{object}' object Permission on '{object_permission_type1}' and '{object_permission_type2}'. Mute the '{field_permission_type1}' for the '{field}' fields.","prerequisites":{"data":{"objects":["PermissionSetGroup"]}},"objects":["PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"permission_set_group":"LaLaLandUsers","object":"Shipment","object_permission_type1":"Create","object_permission_type2":"View All Fields","field_permission_type1":"Edit Access","field":"ExpectedDeliveryDate"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/95"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L708","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_muting_permission_set","data_retrieval_spec":[{"type":"metadata","id":"permission_set_group","params":{"types_and_members":{"PermissionSetGroup":["{permission_set_group.replace(' ', '_')}"]}}},{"type":"metadata","id":"muting_permission_set","params":{"types_and_members":{"MutingPermissionSet":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0a70c9af686f56ebd6761384724b9e110654a9ddddd13e8fd1f2ffc2276544c0"} {"id":"scuba:admin_022_001","source_id":"scuba","original_id":"admin_022_001","kind":"benchmark_task","instruction":"Could you go ahead and grant Foo Bar the MarketingAnalyticsUser permission set group?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_001","query_templated":"Assign the MarketingAnalyticsUser permission set group to user Foo Bar.","query":"Could you go ahead and grant Foo Bar the MarketingAnalyticsUser permission set group?","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/96","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_022","template_name":"assign_permset_group_to_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/96"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/96"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/assign_permset_group_to_user"}],"source_schema_notes":[],"template_registry":{"name":"assign_permset_group_to_user","original":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nCould you go ahead and grant Foo Bar the MarketingAnalyticsUser permission set group?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_001","query_templated":"Assign the MarketingAnalyticsUser permission set group to user Foo Bar.","query":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nCould you go ahead and grant Foo Bar the MarketingAnalyticsUser permission set group?\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/96"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L299","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_assign_permset_group_to_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.DeveloperName, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{permission_set_group}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"969c3df715f73ad3a274d3cf662ea9955c93b00182645e3e6eb7921c310b1b55"} {"id":"scuba:admin_022_002","source_id":"scuba","original_id":"admin_022_002","kind":"benchmark_task","instruction":"I need Sample User added to the Manufacturing Admin permission set group.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_002","query_templated":"Assign the Manufacturing Admin permission set group to user Sample User.","query":"I need Sample User added to the Manufacturing Admin permission set group.","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"Manufacturing Admin","user_name":"Sample User"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"Manufacturing Admin","user_name":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/97","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_022","template_name":"assign_permset_group_to_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/97"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/97"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/assign_permset_group_to_user"}],"source_schema_notes":[],"template_registry":{"name":"assign_permset_group_to_user","original":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nI need Sample User added to the Manufacturing Admin permission set group.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_002","query_templated":"Assign the Manufacturing Admin permission set group to user Sample User.","query":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nI need Sample User added to the Manufacturing Admin permission set group.\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"Manufacturing Admin","user_name":"Sample User"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"Manufacturing Admin","user_name":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/97"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L299","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_assign_permset_group_to_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.DeveloperName, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{permission_set_group}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"fd72add6c0b0ae70d8025657681f3444c065770872ff183567668b636d7f16c0"} {"id":"scuba:admin_022_003","source_id":"scuba","original_id":"admin_022_003","kind":"benchmark_task","instruction":"Attach the MarketingAnalyticsUser permission set group to Foo Bar’s user account.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_003","query_templated":"Assign the MarketingAnalyticsUser permission set group to user Foo Bar.","query":"Attach the MarketingAnalyticsUser permission set group to Foo Bar’s user account.","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/98","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_022","template_name":"assign_permset_group_to_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/98"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/98"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/assign_permset_group_to_user"}],"source_schema_notes":[],"template_registry":{"name":"assign_permset_group_to_user","original":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nAttach the MarketingAnalyticsUser permission set group to Foo Bar’s user account.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_003","query_templated":"Assign the MarketingAnalyticsUser permission set group to user Foo Bar.","query":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nAttach the MarketingAnalyticsUser permission set group to Foo Bar’s user account.\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"MarketingAnalyticsUser","user_name":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/98"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L299","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_assign_permset_group_to_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.DeveloperName, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{permission_set_group}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7822637f54607b84bbdc57ae9e6f95cf96be0b5370be46f78864b48d976e6168"} {"id":"scuba:admin_022_004","source_id":"scuba","original_id":"admin_022_004","kind":"benchmark_task","instruction":"Please provision Sofia Bennett with the SamplePermsetGroup permission set group.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_004","query_templated":"Assign the SamplePermsetGroup permission set group to user Sofia Bennett.","query":"Please provision Sofia Bennett with the SamplePermsetGroup permission set group.","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"SamplePermsetGroup","user_name":"Sofia Bennett"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"SamplePermsetGroup","user_name":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/99","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_022","template_name":"assign_permset_group_to_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/99"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/99"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/assign_permset_group_to_user"}],"source_schema_notes":[],"template_registry":{"name":"assign_permset_group_to_user","original":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nPlease provision Sofia Bennett with the SamplePermsetGroup permission set group.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_004","query_templated":"Assign the SamplePermsetGroup permission set group to user Sofia Bennett.","query":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nPlease provision Sofia Bennett with the SamplePermsetGroup permission set group.\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"SamplePermsetGroup","user_name":"Sofia Bennett"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"SamplePermsetGroup","user_name":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/99"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L299","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_assign_permset_group_to_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.DeveloperName, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{permission_set_group}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3882b2e65b5a18d440b580c1fbe30ef215ab9896614137a18c0ea92bf2e66306"} {"id":"scuba:admin_022_005","source_id":"scuba","original_id":"admin_022_005","kind":"benchmark_task","instruction":"Apply the LaLaLandUsers permission set group to John Doe’s account.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_005","query_templated":"Assign the LaLaLandUsers permission set group to user John Doe.","query":"Apply the LaLaLandUsers permission set group to John Doe’s account.","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"LaLaLandUsers","user_name":"John Doe"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"LaLaLandUsers","user_name":"John Doe"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/100","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_022","template_name":"assign_permset_group_to_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/100"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/100"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/assign_permset_group_to_user"}],"source_schema_notes":[],"template_registry":{"name":"assign_permset_group_to_user","original":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/assign_permset_group_to_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nApply the LaLaLandUsers permission set group to John Doe’s account.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_022_005","query_templated":"Assign the LaLaLandUsers permission set group to user John Doe.","query":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted).\n\n## Your ultimate task\n```\nApply the LaLaLandUsers permission set group to John Doe’s account.\n```","human_review_comment":"accept","ground_truth_dict":{"permission_set_group":"LaLaLandUsers","user_name":"John Doe"},"difficulty":"easy","query_template_name":"assign_permset_group_to_user","query_template_metadata":{"template_id":"admin_022","template_string":"Assign the {permission_set_group} permission set group to user {user_name}.","prerequisites":{"data":{"objects":["User","PermissionSetGroup"]}},"objects":["PermissionSetAssignment"],"metadata_types":[]},"instance_dict":{"permission_set_group":"LaLaLandUsers","user_name":"John Doe"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Assign the [MarketAnalyticsUser] permission set group to the user [Foo Bar].\nEXPERIENCE:\nSuccessful plan to assign the MarketingAnalyticsUser permission set group to user Foo Bar:\n\n- Log in to Salesforce.\n- Open Setup.\n- In Quick Find, type “permission set group” and open Permission Set Groups.\n- Select the “MarketingAnalyticsUser” permission set group.\n- Click Manage Assignments, then Add Assignments.\n- Select the user “Foo Bar” (open the user record).\n- In the user record, open Permission Set Group Assignments, click Edit Assignments.\n- Select “MarketingAnalyticsUser” from Available groups (scroll if needed), click Add.\n- Click Save, then Continue if prompted.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Assign the [Manufacturing Admin] permission set group to the user [Sample user].\n\nEXPERIENCE:\nSuccessful plan to assign the Manufacturing Admin permission set group to Sample User:\n\n1. Log in and open Setup.\n2. In Quick Find, type “permission set group” and open Permission Set Groups.\n3. Open the “Manufacturing Admin” permission set group.\n4. Click Manage Assignments > Add Assignment.\n5. Select “Sample User” to open the user record.\n6. In the user’s Permission Set Group Assignments related list, click Edit Assignments.\n7. Move “Manufacturing Admin” from Available to Enabled (click Add).\n8. Click Save and confirm (Continue if prompted)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/100"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L299","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_assign_permset_group_to_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.DeveloperName, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{permission_set_group}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e497f524d363bae90614be1a6d97bf11614086abfd30d25ed20261e89730cd86"} {"id":"scuba:admin_023_001","source_id":"scuba","original_id":"admin_023_001","kind":"benchmark_task","instruction":"For MyAnimal__c, tighten sharing: set the organization-wide defaults to Private for internal users and Private for external users, and make sure Grant Access Using Hierarchies is turned on.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_001","query_templated":"Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.","query":"For MyAnimal__c, tighten sharing: set the organization-wide defaults to Private for internal users and Private for external users, and make sure Grant Access Using Hierarchies is turned on.","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyAnimal__c","internal_visibility":"Private","external_visibility":"Private"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyAnimal__c","internal_visibility":"Private","external_visibility":"Private"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/101","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_023","template_name":"create_owd_settings","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/101"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/101"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_owd_settings"}],"source_schema_notes":[],"template_registry":{"name":"create_owd_settings","original":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nFor MyAnimal__c, tighten sharing: set the organization-wide defaults to Private for internal users and Private for external users, and make sure Grant Access Using Hierarchies is turned on.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_001","query_templated":"Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nFor MyAnimal__c, tighten sharing: set the organization-wide defaults to Private for internal users and Private for external users, and make sure Grant Access Using Hierarchies is turned on.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyAnimal__c","internal_visibility":"Private","external_visibility":"Private"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyAnimal__c","internal_visibility":"Private","external_visibility":"Private"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/101"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L751","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_owd_settings","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0e0baa6150b13340dcb63c23c77ef418fe4cbdf0472f7227ec8a7acc0b7c96cb"} {"id":"scuba:admin_023_002","source_id":"scuba","original_id":"admin_023_002","kind":"benchmark_task","instruction":"Set the organization-wide defaults for the MyBike__c object to Public Read Only for internal users, Private for external users, and enable access for users higher in the role hierarchy.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_002","query_templated":"Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.","query":"Set the organization-wide defaults for the MyBike__c object to Public Read Only for internal users, Private for external users, and enable access for users higher in the role hierarchy.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"MyBike__c","internal_visibility":"Read","external_visibility":"Private"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyBike__c","internal_visibility":"Public Read Only","external_visibility":"Private"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/102","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_023","template_name":"create_owd_settings","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/102"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/102"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_owd_settings"}],"source_schema_notes":[],"template_registry":{"name":"create_owd_settings","original":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nSet the organization-wide defaults for the MyBike__c object to Public Read Only for internal users, Private for external users, and enable access for users higher in the role hierarchy.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_002","query_templated":"Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nSet the organization-wide defaults for the MyBike__c object to Public Read Only for internal users, Private for external users, and enable access for users higher in the role hierarchy.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"MyBike__c","internal_visibility":"Read","external_visibility":"Private"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyBike__c","internal_visibility":"Public Read Only","external_visibility":"Private"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/102"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L751","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_owd_settings","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"06a4765bb08b033cd2615cd1909f85c9052dfa8388dc99fd43758e3eb5cd23ff"} {"id":"scuba:admin_023_003","source_id":"scuba","original_id":"admin_023_003","kind":"benchmark_task","instruction":"Set the organization-wide defaults for the Environment__c object to Public Read/Write for internal users, Public Read Only for external users, and allow access for users higher in the role hierarchy.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_003","query_templated":"Configure the organization-wide defaults for the Environment__c object with visibility Public Read/Write for internal users, visibility Public Read Only for external users, and allow users higher in the role hierarchy.","query":"Set the organization-wide defaults for the Environment__c object to Public Read/Write for internal users, Public Read Only for external users, and allow access for users higher in the role hierarchy.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Environment__c","internal_visibility":"ReadWrite","external_visibility":"Read"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Environment__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read Only"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/103","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_023","template_name":"create_owd_settings","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/103"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/103"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_owd_settings"}],"source_schema_notes":[],"template_registry":{"name":"create_owd_settings","original":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nSet the organization-wide defaults for the Environment__c object to Public Read/Write for internal users, Public Read Only for external users, and allow access for users higher in the role hierarchy.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_003","query_templated":"Configure the organization-wide defaults for the Environment__c object with visibility Public Read/Write for internal users, visibility Public Read Only for external users, and allow users higher in the role hierarchy.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nSet the organization-wide defaults for the Environment__c object to Public Read/Write for internal users, Public Read Only for external users, and allow access for users higher in the role hierarchy.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"Environment__c","internal_visibility":"ReadWrite","external_visibility":"Read"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"Environment__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read Only"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/103"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L751","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_owd_settings","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"033aed6e8a304113daa7e84b455d1b2378cbe4bd8b2637a8e988d0ae858953b0"} {"id":"scuba:admin_023_004","source_id":"scuba","original_id":"admin_023_004","kind":"benchmark_task","instruction":"Configure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_004","query_templated":"Configure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.","query":"Configure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"MyInventory__c","internal_visibility":"ReadWrite","external_visibility":"ReadWrite"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyInventory__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read/Write"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/104","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_023","template_name":"create_owd_settings","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/104"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/104"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_owd_settings"}],"source_schema_notes":[],"template_registry":{"name":"create_owd_settings","original":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nConfigure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_004","query_templated":"Configure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nConfigure the organization-wide defaults for the MyInventory__c object with visibility Public Read/Write for internal users, visibility Public Read/Write for external users, and allow users higher in the role hierarchy.\n```","human_review_comment":"rewrite","ground_truth_dict":{"object_name":"MyInventory__c","internal_visibility":"ReadWrite","external_visibility":"ReadWrite"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyInventory__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read/Write"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/104"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L751","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_owd_settings","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b94559f0a15846a8b6243d3f50a5392e3068604e59d2d0338c4285d055467d45"} {"id":"scuba:admin_023_005","source_id":"scuba","original_id":"admin_023_005","kind":"benchmark_task","instruction":"Please configure MyVehicle__c’s organization-wide defaults to Public Read/Write for internal users and Public Read Only for external users, and ensure access is still granted to users higher in the role hierarchy.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_005","query_templated":"Configure the organization-wide defaults for the MyVehicle__c object with visibility Public Read/Write for internal users, visibility Public Read Only for external users, and allow users higher in the role hierarchy.","query":"Please configure MyVehicle__c’s organization-wide defaults to Public Read/Write for internal users and Public Read Only for external users, and ensure access is still granted to users higher in the role hierarchy.","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyVehicle__c","internal_visibility":"ReadWrite","external_visibility":"Read"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyVehicle__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read Only"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/105","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"easy","template_id":"admin_023","template_name":"create_owd_settings","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/105"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/105"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_owd_settings"}],"source_schema_notes":[],"template_registry":{"name":"create_owd_settings","original":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_owd_settings","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nPlease configure MyVehicle__c’s organization-wide defaults to Public Read/Write for internal users and Public Read Only for external users, and ensure access is still granted to users higher in the role hierarchy.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_023_005","query_templated":"Configure the organization-wide defaults for the MyVehicle__c object with visibility Public Read/Write for internal users, visibility Public Read Only for external users, and allow users higher in the role hierarchy.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\n## Your ultimate task\n```\nPlease configure MyVehicle__c’s organization-wide defaults to Public Read/Write for internal users and Public Read Only for external users, and ensure access is still granted to users higher in the role hierarchy.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"MyVehicle__c","internal_visibility":"ReadWrite","external_visibility":"Read"},"difficulty":"easy","query_template_name":"create_owd_settings","query_template_metadata":{"template_id":"admin_023","template_string":"Configure the organization-wide defaults for the {object_name} object with visibility {internal_visibility} for internal users, visibility {external_visibility} for external users, and allow users higher in the role hierarchy.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["MyAnimal__c","MyBike__c","Environment__c","MyInventory__c","MyVehicle__c"]}}},"objects":[],"metadata_types":["CustomObject"]},"instance_dict":{"object_name":"MyVehicle__c","internal_visibility":"Public Read/Write","external_visibility":"Public Read Only"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the organization-wide defaults for the MyAnimal__c object with visibility Private for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan to configure OWD for MyAnimal__c:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “Sharing Settings” and open Sharing Settings.\n- Click Edit in the Organization-Wide Defaults section.\n- Scroll to the MyAnimal__c row.\n- Set Default Internal Access to Private.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Configure the organization-wide defaults for the MyBike__c object with visibility Public Read Only for internal users, visibility Private for external users, and allow users higher in the role hierarchy.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Quick Find, enter “Sharing Settings” and open it.\n- Click Edit under Organization-Wide Defaults.\n- Scroll to/find the MyBike__c object row.\n- Set Default Internal Access to Public Read Only.\n- Set Default External Access to Private.\n- Check Grant Access Using Hierarchies (to allow users higher in the role hierarchy).\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/105"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L751","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_owd_settings","data_retrieval_spec":[{"type":"metadata","id":"custom_objects","params":{"types_and_members":{"CustomObject":["{object_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4a8885c6767d4505171cd6aff673f220215526454b1a97477057014e5de94ea0"} {"id":"scuba:admin_025_001","source_id":"scuba","original_id":"admin_025_001","kind":"benchmark_task","instruction":"Please freeze the account for the Username \"Alice Bob\" right away.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_001","query_templated":"Freeze a user with Username Alice Bob","query":"Please freeze the account for the Username \"Alice Bob\" right away.","human_review_comment":"accept","ground_truth_dict":{"username":"Alice Bob"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/106","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"medium","template_id":"admin_025","template_name":"freeze_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/106"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/106"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/freeze_user"}],"source_schema_notes":[],"template_registry":{"name":"freeze_user","original":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease freeze the account for the Username \"Alice Bob\" right away.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_001","query_templated":"Freeze a user with Username Alice Bob","query":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease freeze the account for the Username \"Alice Bob\" right away.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Alice Bob"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/106"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L96","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_freeze_user","data_retrieval_spec":[{"type":"soql","id":"user_info","params":{"query":"SELECT Id FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}},{"type":"soql","id":"user_login_status","params":{"query":"SELECT IsFrozen FROM UserLogin WHERE UserId='{retrieved_results['user_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0feca1bbe2ab0bd620d886980578f6c4b7c8e664f60ccdc03fc9e86ed102b889"} {"id":"scuba:admin_025_002","source_id":"scuba","original_id":"admin_025_002","kind":"benchmark_task","instruction":"Go ahead and put a freeze on the user with Username Foo Bar, ASAP.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_002","query_templated":"Freeze a user with Username Foo Bar","query":"Go ahead and put a freeze on the user with Username Foo Bar, ASAP.","human_review_comment":"accept","ground_truth_dict":{"username":"Foo Bar"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/107","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"medium","template_id":"admin_025","template_name":"freeze_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/107"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/107"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/freeze_user"}],"source_schema_notes":[],"template_registry":{"name":"freeze_user","original":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nGo ahead and put a freeze on the user with Username Foo Bar, ASAP.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_002","query_templated":"Freeze a user with Username Foo Bar","query":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nGo ahead and put a freeze on the user with Username Foo Bar, ASAP.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Foo Bar"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/107"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L96","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_freeze_user","data_retrieval_spec":[{"type":"soql","id":"user_info","params":{"query":"SELECT Id FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}},{"type":"soql","id":"user_login_status","params":{"query":"SELECT IsFrozen FROM UserLogin WHERE UserId='{retrieved_results['user_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"401193a5dd4aabc63ace78e81f51787b507d39c0fd6f8225fb6c18d83687f968"} {"id":"scuba:admin_025_003","source_id":"scuba","original_id":"admin_025_003","kind":"benchmark_task","instruction":"I need the user account whose Username is John Doe to be frozen—thanks.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_003","query_templated":"Freeze a user with Username John Doe","query":"I need the user account whose Username is John Doe to be frozen—thanks.","human_review_comment":"accept","ground_truth_dict":{"username":"John Doe"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/108","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"medium","template_id":"admin_025","template_name":"freeze_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/108"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/108"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/freeze_user"}],"source_schema_notes":[],"template_registry":{"name":"freeze_user","original":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI need the user account whose Username is John Doe to be frozen—thanks.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_003","query_templated":"Freeze a user with Username John Doe","query":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI need the user account whose Username is John Doe to be frozen—thanks.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"John Doe"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/108"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L96","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_freeze_user","data_retrieval_spec":[{"type":"soql","id":"user_info","params":{"query":"SELECT Id FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}},{"type":"soql","id":"user_login_status","params":{"query":"SELECT IsFrozen FROM UserLogin WHERE UserId='{retrieved_results['user_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"cbaeaeb666f9ac91aec80542f26fd7fabe33d0f40ec89ff8af5cb6e44af4bfdd"} {"id":"scuba:admin_025_004","source_id":"scuba","original_id":"admin_025_004","kind":"benchmark_task","instruction":"Lock it down: place a freeze on the profile for the Username Sofia Bennett.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_004","query_templated":"Freeze a user with Username Sofia Bennett","query":"Lock it down: place a freeze on the profile for the Username Sofia Bennett.","human_review_comment":"accept","ground_truth_dict":{"username":"Sofia Bennett"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/109","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"medium","template_id":"admin_025","template_name":"freeze_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/109"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/109"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/freeze_user"}],"source_schema_notes":[],"template_registry":{"name":"freeze_user","original":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nLock it down: place a freeze on the profile for the Username Sofia Bennett.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_004","query_templated":"Freeze a user with Username Sofia Bennett","query":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nLock it down: place a freeze on the profile for the Username Sofia Bennett.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Sofia Bennett"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/109"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L96","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_freeze_user","data_retrieval_spec":[{"type":"soql","id":"user_info","params":{"query":"SELECT Id FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}},{"type":"soql","id":"user_login_status","params":{"query":"SELECT IsFrozen FROM UserLogin WHERE UserId='{retrieved_results['user_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"dfab584c95475bed0c6ca74059472a0c28e96c43b6f8bfa75fa17983ea0265e1"} {"id":"scuba:admin_025_005","source_id":"scuba","original_id":"admin_025_005","kind":"benchmark_task","instruction":"Initiate a freeze on the user tied to the Username Sample User.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_005","query_templated":"Freeze a user with Username Sample User","query":"Initiate a freeze on the user tied to the Username Sample User.","human_review_comment":"accept","ground_truth_dict":{"username":"Sample User"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/110","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"medium","template_id":"admin_025","template_name":"freeze_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/110"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/110"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/freeze_user"}],"source_schema_notes":[],"template_registry":{"name":"freeze_user","original":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/freeze_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nInitiate a freeze on the user tied to the Username Sample User.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_025_005","query_templated":"Freeze a user with Username Sample User","query":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nInitiate a freeze on the user tied to the Username Sample User.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Sample User"},"difficulty":"medium","query_template_name":"freeze_user","query_template_metadata":{"template_id":"admin_025","template_string":"Freeze a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["UserLogin"],"metadata_types":[]},"instance_dict":{"username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Freeze a user with Username Alice Bob\nEXPERIENCE:\nSuccessful plan:\n- Log in to the system by clicking Log In.\n- Use the Global Search Bar: click it and type \"alice bob\".\n- From results, select \"Alice Bob — User\".\n- Open the User Detail page.\n- Click Freeze to freeze the user.\nSample 2:\nTASK:Freeze a user with Username Foo Bar\nEXPERIENCE:\nSuccessful plan to freeze user \"Foo Bar\":\n\n1. Log in by clicking the Log In button.\n2. Use the Global Search Bar to search for \"foo bar\".\n3. From the results, click the \"Foo Bar\" User entry.\n4. Open the User Detail page.\n5. Click the Freeze action to freeze the user.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/110"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L96","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_freeze_user","data_retrieval_spec":[{"type":"soql","id":"user_info","params":{"query":"SELECT Id FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}},{"type":"soql","id":"user_login_status","params":{"query":"SELECT IsFrozen FROM UserLogin WHERE UserId='{retrieved_results['user_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"654efcc662045198d73a55a671fa9cfeeaa488e4a735f3e01fb8d524ab637bcf"} {"id":"scuba:admin_026_001","source_id":"scuba","original_id":"admin_026_001","kind":"benchmark_task","instruction":"Please disable the account for the user whose username is \"Sample User\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_001","query_templated":"Deactivate a user with Username Sample User","query":"Please disable the account for the user whose username is \"Sample User\"","human_review_comment":"accept","ground_truth_dict":{"username":"Sample User"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/111","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"easy","template_id":"admin_026","template_name":"deactivate_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/111"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/111"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/deactivate_user"}],"source_schema_notes":[],"template_registry":{"name":"deactivate_user","original":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease disable the account for the user whose username is \"Sample User\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_001","query_templated":"Deactivate a user with Username Sample User","query":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease disable the account for the user whose username is \"Sample User\"\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Sample User"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/111"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L82","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_deactivate_user","data_retrieval_spec":[{"type":"soql","id":"user_status","params":{"query":"SELECT IsActive FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7eadd3e4d11d1cb149265b91a3d17e3526172db2ec091a1728af8d94095cd6e7"} {"id":"scuba:admin_026_002","source_id":"scuba","original_id":"admin_026_002","kind":"benchmark_task","instruction":"Could you deactivate the user account for John Doe (username: John Doe), please?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_002","query_templated":"Deactivate a user with Username John Doe","query":"Could you deactivate the user account for John Doe (username: John Doe), please?","human_review_comment":"accept","ground_truth_dict":{"username":"John Doe"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/112","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"easy","template_id":"admin_026","template_name":"deactivate_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/112"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/112"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/deactivate_user"}],"source_schema_notes":[],"template_registry":{"name":"deactivate_user","original":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you deactivate the user account for John Doe (username: John Doe), please?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_002","query_templated":"Deactivate a user with Username John Doe","query":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you deactivate the user account for John Doe (username: John Doe), please?\n```","human_review_comment":"accept","ground_truth_dict":{"username":"John Doe"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/112"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L82","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_deactivate_user","data_retrieval_spec":[{"type":"soql","id":"user_status","params":{"query":"SELECT IsActive FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f37234dd6bf8cb7984f2e199c05fc5d45dbb7f25d52978d7cf92dee621793a52"} {"id":"scuba:admin_026_003","source_id":"scuba","original_id":"admin_026_003","kind":"benchmark_task","instruction":"ASAP, mark the user with the username \"Foo Bar\" as inactive—deactivate that account.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_003","query_templated":"Deactivate a user with Username Foo Bar","query":"ASAP, mark the user with the username \"Foo Bar\" as inactive—deactivate that account.","human_review_comment":"accept","ground_truth_dict":{"username":"Foo Bar"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/113","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"easy","template_id":"admin_026","template_name":"deactivate_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/113"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/113"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/deactivate_user"}],"source_schema_notes":[],"template_registry":{"name":"deactivate_user","original":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nASAP, mark the user with the username \"Foo Bar\" as inactive—deactivate that account.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_003","query_templated":"Deactivate a user with Username Foo Bar","query":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nASAP, mark the user with the username \"Foo Bar\" as inactive—deactivate that account.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Foo Bar"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/113"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L82","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_deactivate_user","data_retrieval_spec":[{"type":"soql","id":"user_status","params":{"query":"SELECT IsActive FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9ea156080f4e107025ffce231d41059a06afb6b1d69708b0b5ed5abd2579c0f1"} {"id":"scuba:admin_026_004","source_id":"scuba","original_id":"admin_026_004","kind":"benchmark_task","instruction":"For the user named Hello World, flip their status to inactive and fully deactivate the account.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_004","query_templated":"Deactivate a user with Username Hello World","query":"For the user named Hello World, flip their status to inactive and fully deactivate the account.","human_review_comment":"accept","ground_truth_dict":{"username":"Hello World"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Hello World"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/114","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"easy","template_id":"admin_026","template_name":"deactivate_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/114"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/114"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/deactivate_user"}],"source_schema_notes":[],"template_registry":{"name":"deactivate_user","original":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the user named Hello World, flip their status to inactive and fully deactivate the account.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_004","query_templated":"Deactivate a user with Username Hello World","query":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the user named Hello World, flip their status to inactive and fully deactivate the account.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Hello World"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Hello World"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/114"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L82","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_deactivate_user","data_retrieval_spec":[{"type":"soql","id":"user_status","params":{"query":"SELECT IsActive FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"904e152a43d2b17b0e7198259d61a05b5bed5c974bdc240ce2e59914df39a431"} {"id":"scuba:admin_026_005","source_id":"scuba","original_id":"admin_026_005","kind":"benchmark_task","instruction":"Deactivate the account tied to the username \"Alice Bob\" so this user can no longer sign in.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_005","query_templated":"Deactivate a user with Username Alice Bob","query":"Deactivate the account tied to the username \"Alice Bob\" so this user can no longer sign in.","human_review_comment":"accept","ground_truth_dict":{"username":"Alice Bob"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/115","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Users","source_difficulty":"easy","template_id":"admin_026","template_name":"deactivate_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/115"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/115"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/deactivate_user"}],"source_schema_notes":[],"template_registry":{"name":"deactivate_user","original":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/deactivate_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nDeactivate the account tied to the username \"Alice Bob\" so this user can no longer sign in.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Users","task_id":"admin_026_005","query_templated":"Deactivate a user with Username Alice Bob","query":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nDeactivate the account tied to the username \"Alice Bob\" so this user can no longer sign in.\n```","human_review_comment":"accept","ground_truth_dict":{"username":"Alice Bob"},"difficulty":"easy","query_template_name":"deactivate_user","query_template_metadata":{"template_id":"admin_026","template_string":"Deactivate a user with Username {username}","prerequisites":{"data":{"objects":["User"]}},"objects":["User"],"metadata_types":[]},"instance_dict":{"username":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Deactivate a user with Username Sample User\nEXPERIENCE:\nSuccessful plan to deactivate user \"Sample User\":\n\n- Log in to the application (if prompted).\n- Open Setup (gear icon > Setup).\n- In Quick Find, type \"Users\" and select Users.\n- Locate and open the user record for \"Sample User,\" then click Edit.\n- Uncheck the “Active” checkbox to deactivate the user.\n- Confirm the deactivation in the prompt (click OK).\n- Click Save to apply the change.\nSample 2:\nTASK:Deactivate a user with Username John Doe\nEXPERIENCE:\nSuccessful plan to deactivate user \"John Doe\":\n\n1) Log in to the application.\n2) Open Setup.\n3) In Quick Find, type “Users” and select the Users page.\n4) Locate user “John Doe” and click Edit.\n5) Toggle off the “Active” checkbox.\n6) Confirm the deactivation when prompted (OK).\n7) Click Save to apply changes.\n\nNotes:\n- Input used: typed “Users” into Quick Find.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/115"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L82","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_deactivate_user","data_retrieval_spec":[{"type":"soql","id":"user_status","params":{"query":"SELECT IsActive FROM User WHERE User.FirstName='{username.split()[0]}' AND User.LastName='{username.split()[1]}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8d89f37bb17fab40184281e4c55991d2ff951fb8cd9c3cacdee4f9f0d34889c7"} {"id":"scuba:admin_028_001","source_id":"scuba","original_id":"admin_028_001","kind":"benchmark_task","instruction":"Could you create a permission set group called AgentforceReadOnlyAccess, drop in the Read-Only and Analytics View Access permissions, and then assign that group to John Doe?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_001","query_templated":"Create a new permission set group AgentforceReadOnlyAccess and add permissions Read-Only, Analytics View Access to this group, assign this permission set group to user John Doe.","query":"Could you create a permission set group called AgentforceReadOnlyAccess, drop in the Read-Only and Analytics View Access permissions, and then assign that group to John Doe?","human_review_comment":"accept","ground_truth_dict":{"group_name":"AgentforceReadOnlyAccess","comma_separated_permissions_list":"Read-Only, Analytics View Access","username":"John Doe"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"AgentforceReadOnlyAccess","comma_separated_permissions_list":"Read-Only, Analytics View Access","username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/116","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_028","template_name":"create_permset_group_assign_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/116"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/116"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_assign_user"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_assign_user","original":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nCould you create a permission set group called AgentforceReadOnlyAccess, drop in the Read-Only and Analytics View Access permissions, and then assign that group to John Doe?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_001","query_templated":"Create a new permission set group AgentforceReadOnlyAccess and add permissions Read-Only, Analytics View Access to this group, assign this permission set group to user John Doe.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nCould you create a permission set group called AgentforceReadOnlyAccess, drop in the Read-Only and Analytics View Access permissions, and then assign that group to John Doe?\n```","human_review_comment":"accept","ground_truth_dict":{"group_name":"AgentforceReadOnlyAccess","comma_separated_permissions_list":"Read-Only, Analytics View Access","username":"John Doe"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"AgentforceReadOnlyAccess","comma_separated_permissions_list":"Read-Only, Analytics View Access","username":"John Doe"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/116"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L33","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_assign_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_info","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.MasterLabel, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"beb5915e398d2c9bbd8839bf856d2c7d7583503964172bdac70e94275760c986"} {"id":"scuba:admin_028_002","source_id":"scuba","original_id":"admin_028_002","kind":"benchmark_task","instruction":"Please create a permission set group named MarketingInsightsUserGroup, include the Marketing Manager and Dashboard Viewer permissions, and assign the group to the user Sample User.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_002","query_templated":"Create a new permission set group MarketingInsightsUserGroup and add permissions Marketing Manager, Dashboard Viewer to this group, assign this permission set group to user Sample User.","query":"Please create a permission set group named MarketingInsightsUserGroup, include the Marketing Manager and Dashboard Viewer permissions, and assign the group to the user Sample User.","human_review_comment":"change spin up","ground_truth_dict":{"group_name":"MarketingInsightsUserGroup","comma_separated_permissions_list":"Marketing Manager, Dashboard Viewer","username":"Sample User"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"MarketingInsightsUserGroup","comma_separated_permissions_list":"Marketing Manager, Dashboard Viewer","username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/117","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_028","template_name":"create_permset_group_assign_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/117"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/117"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_assign_user"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_assign_user","original":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nPlease create a permission set group named MarketingInsightsUserGroup, include the Marketing Manager and Dashboard Viewer permissions, and assign the group to the user Sample User.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_002","query_templated":"Create a new permission set group MarketingInsightsUserGroup and add permissions Marketing Manager, Dashboard Viewer to this group, assign this permission set group to user Sample User.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nPlease create a permission set group named MarketingInsightsUserGroup, include the Marketing Manager and Dashboard Viewer permissions, and assign the group to the user Sample User.\n```","human_review_comment":"change spin up","ground_truth_dict":{"group_name":"MarketingInsightsUserGroup","comma_separated_permissions_list":"Marketing Manager, Dashboard Viewer","username":"Sample User"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"MarketingInsightsUserGroup","comma_separated_permissions_list":"Marketing Manager, Dashboard Viewer","username":"Sample User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/117"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L33","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_assign_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_info","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.MasterLabel, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"52afd0bc062bf7f3c3e44681e0e5ff24642cbe80c7f5b35a38948431070779da"} {"id":"scuba:admin_028_003","source_id":"scuba","original_id":"admin_028_003","kind":"benchmark_task","instruction":"I need this ASAP: create the permission set group ServiceOperationsAccess, add the CRM User and Service Cloud User permissions, and attach that group to Foo Bar.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_003","query_templated":"Create a new permission set group ServiceOperationsAccess and add permissions CRM User, Service Cloud User to this group, assign this permission set group to user Foo Bar.","query":"I need this ASAP: create the permission set group ServiceOperationsAccess, add the CRM User and Service Cloud User permissions, and attach that group to Foo Bar.","human_review_comment":"accept","ground_truth_dict":{"group_name":"ServiceOperationsAccess","comma_separated_permissions_list":"CRM User, Service Cloud User","username":"Foo Bar"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"ServiceOperationsAccess","comma_separated_permissions_list":"CRM User, Service Cloud User","username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/118","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_028","template_name":"create_permset_group_assign_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/118"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/118"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_assign_user"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_assign_user","original":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nI need this ASAP: create the permission set group ServiceOperationsAccess, add the CRM User and Service Cloud User permissions, and attach that group to Foo Bar.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_003","query_templated":"Create a new permission set group ServiceOperationsAccess and add permissions CRM User, Service Cloud User to this group, assign this permission set group to user Foo Bar.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nI need this ASAP: create the permission set group ServiceOperationsAccess, add the CRM User and Service Cloud User permissions, and attach that group to Foo Bar.\n```","human_review_comment":"accept","ground_truth_dict":{"group_name":"ServiceOperationsAccess","comma_separated_permissions_list":"CRM User, Service Cloud User","username":"Foo Bar"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"ServiceOperationsAccess","comma_separated_permissions_list":"CRM User, Service Cloud User","username":"Foo Bar"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/118"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L33","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_assign_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_info","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.MasterLabel, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3c1ee804648a9c6b8f5c96d78414bc077f321ac92f68ec4e47cbb92c64cdb11f"} {"id":"scuba:admin_028_004","source_id":"scuba","original_id":"admin_028_004","kind":"benchmark_task","instruction":"First create a new permission set group called FinanceTeamAccess. Next, add the Order Management Agent, and Buyer permissions. Finally, assign this group to Maria Rodriguez.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_004","query_templated":"Create a new permission set group FinanceTeamAccess and add permissions Order Management Agent, Buyer to this group, assign this permission set group to user Maria Rodriguez.","query":"First create a new permission set group called FinanceTeamAccess. Next, add the Order Management Agent, and Buyer permissions. Finally, assign this group to Maria Rodriguez.","human_review_comment":"add comma","ground_truth_dict":{"group_name":"FinanceTeamAccess","comma_separated_permissions_list":"Order Management Agent, Buyer","username":"Maria Rodriguez"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"FinanceTeamAccess","comma_separated_permissions_list":"Order Management Agent, Buyer","username":"Maria Rodriguez"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/119","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_028","template_name":"create_permset_group_assign_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/119"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/119"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_assign_user"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_assign_user","original":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nFirst create a new permission set group called FinanceTeamAccess. Next, add the Order Management Agent, and Buyer permissions. Finally, assign this group to Maria Rodriguez.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_004","query_templated":"Create a new permission set group FinanceTeamAccess and add permissions Order Management Agent, Buyer to this group, assign this permission set group to user Maria Rodriguez.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nFirst create a new permission set group called FinanceTeamAccess. Next, add the Order Management Agent, and Buyer permissions. Finally, assign this group to Maria Rodriguez.\n```","human_review_comment":"add comma","ground_truth_dict":{"group_name":"FinanceTeamAccess","comma_separated_permissions_list":"Order Management Agent, Buyer","username":"Maria Rodriguez"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"FinanceTeamAccess","comma_separated_permissions_list":"Order Management Agent, Buyer","username":"Maria Rodriguez"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/119"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L33","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_assign_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_info","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.MasterLabel, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c90ee8993d052d4495e65771ffb3dd295209318c245c5273724747195739174a"} {"id":"scuba:admin_028_005","source_id":"scuba","original_id":"admin_028_005","kind":"benchmark_task","instruction":"After you create the permission set group AuditReadOnlyGroup, populate it with the Commerce Admin, and Commerce Session permissions and then grant that group to the user Auditor.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_005","query_templated":"Create a new permission set group AuditReadOnlyGroup and add permissions Commerce Admin, Commerce Session to this group, assign this permission set group to user Auditor.","query":"After you create the permission set group AuditReadOnlyGroup, populate it with the Commerce Admin, and Commerce Session permissions and then grant that group to the user Auditor.","human_review_comment":"add comma","ground_truth_dict":{"group_name":"AuditReadOnlyGroup","comma_separated_permissions_list":"Commerce Admin, Commerce Session","username":"Auditor"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"AuditReadOnlyGroup","comma_separated_permissions_list":"Commerce Admin, Commerce Session","username":"Auditor"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/120","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Permission Sets","source_difficulty":"medium","template_id":"admin_028","template_name":"create_permset_group_assign_user","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/120"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/120"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","sha256":"4278941901571535a8f14eba8798f172a702ff2d0baa2252339dab613a4ecbb4","source_json_pointer":"/create_permset_group_assign_user"}],"source_schema_notes":[],"template_registry":{"name":"create_permset_group_assign_user","original":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info.json","sha256":"ed58549a33a8aa7be921e0ea5c4351c2237c6b6504faa3456e269d5106090792","source_json_pointer":"/create_permset_group_assign_user","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nAfter you create the permission set group AuditReadOnlyGroup, populate it with the Commerce Admin, and Commerce Session permissions and then grant that group to the user Auditor.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Permission Sets","task_id":"admin_028_005","query_templated":"Create a new permission set group AuditReadOnlyGroup and add permissions Commerce Admin, Commerce Session to this group, assign this permission set group to user Auditor.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”).\n\n## Your ultimate task\n```\nAfter you create the permission set group AuditReadOnlyGroup, populate it with the Commerce Admin, and Commerce Session permissions and then grant that group to the user Auditor.\n```","human_review_comment":"add comma","ground_truth_dict":{"group_name":"AuditReadOnlyGroup","comma_separated_permissions_list":"Commerce Admin, Commerce Session","username":"Auditor"},"difficulty":"medium","query_template_name":"create_permset_group_assign_user","query_template_metadata":{"template_id":"admin_028","template_string":"Create a new permission set group {group_name} and add permissions {comma_separated_permissions_list} to this group, assign this permission set group to user {username}.","evaluation_type":"data","prerequisites":{"data":{"objects":["User","PermissionSet"]}},"objects":["PermissionSetAssignment","PermissionSetGroup"],"metadata_types":[]},"instance_dict":{"group_name":"AuditReadOnlyGroup","comma_separated_permissions_list":"Commerce Admin, Commerce Session","username":"Auditor"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new permission set group named \"AgentforceReadOnlyAccess\". Add the permission sets [\"Read-Only\", \"Analytics View Access\"] to the group. Then assign this permission set group to the user \"John Doe\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Setup’s Quick Find, search for “Permission Set Groups” and open it.\n- Create the group:\n - Click New Permission Set Group.\n - Enter Label: AgentforceReadOnlyAccess.\n - Save.\n- Add permission sets to the group:\n - Click Permission Sets in Group > Add Permission Set.\n - Use the alphabetical index “A” to jump to A; select Analytics View Access; click Add, then Done.\n - Click Add Permission Set again.\n - Use the alphabetical index “R” to jump to R; check Read-Only; click Add, then Done.\n- Assign the group to the user John Doe:\n - Back to the Permission Set Group > Manage Assignments > Add Assignment.\n - Select John Doe and open Permission Set Group Assignments.\n - Click Edit Assignments, select AgentforceReadOnlyAccess, click Add, then Save.\n\nNotes:\n- The alphabetical index (“A”, “R”) was used to quickly navigate the permission set list.\n- No keyboard hotkeys were used.\nSample 2:\nTASK:Create a new permission set group named \"MarketingInsightsUserGroup\". Add the permission sets [\"Marketing Manager\", \"Dashboard Viewer\"] to the group. Assign this permission set group to the user \"Sample User\".\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Permission Set Groups\n- Log in, open Setup.\n- In Quick Find, type “permission set group” and open “Permission Set Groups”.\n\n2) Create the group\n- Click “New Permission Set Group”.\n- Enter Label: “MarketingInsightsUserGroup”.\n- Save.\n\n3) Add required permission sets to the group\n- Open “Permission Sets in Group” > “Add Permission Set”.\n- Use the letter index “M”, select “Marketing Manager”, click “Add”, then “Done”.\n- Click “Add Permission Set” again.\n- Use the letter index “D”, select “Dashboard Viewer”, click “Add”, then “Done”.\n\n4) Assign the group to the user “Sample User”\n- Go back to the Permission Set Group detail page.\n- Click “Manage Assignments” > “Add Assignment”.\n- Select “Sample User” to open the user record.\n- In the user record, open “Permission Set Group Assignments” > “Edit Assignments”.\n- Select “MarketingInsightsUserGroup”, click “Add”, then “Save”.\n\nNotes:\n- No hotkeys were used; navigation was via clicks, typing in Quick Find, and list letter indexes (“M” and “D”)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/120"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin.py#L33","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_permset_group_assign_user","data_retrieval_spec":[{"type":"soql","id":"permission_set_group_info","params":{"query":"SELECT Id FROM PermissionSetGroup WHERE MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_components","params":{"query":"SELECT PermissionSetGroup.Id,PermissionSetGroup.MasterLabel, PermissionSet.Label FROM PermissionSetGroupComponent WHERE PermissionSetGroup.MasterLabel='{group_name}'"}},{"type":"soql","id":"permission_set_group_assignments","params":{"query":"SELECT PermissionSetGroup.MasterLabel, Assignee.FirstName, Assignee.LastName FROM PermissionSetAssignment WHERE PermissionSetGroup.MasterLabel='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"98aef674ed2dc1fac44699ffc9013f1d8f334a6abf4cda5108bc909ded8f3751"} {"id":"scuba:admin_029_001","source_id":"scuba","original_id":"admin_029_001","kind":"benchmark_task","instruction":"On the Opportunity object, please add a formula field named “Days to close” that shows how many days are left until the Close Date. Lock down field visibility so only the Marketing User profile can see it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_001","query_templated":"Add a formula field with the name Days to close to the object Opportunity. It tracks the number of days until an Opportunity Closes. This newly added field should only be visible to Marketing User profiles.","query":"On the Opportunity object, please add a formula field named “Days to close” that shows how many days are left until the Close Date. Lock down field visibility so only the Marketing User profile can see it.","human_review_comment":"accept","ground_truth_dict":{"field_name":"Days_to_close__c","object_name":"Opportunity","formula":"CloseDate - TODAY()","comma_separated_profiles":"MarketingProfile"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Days to close","object_name":"Opportunity","field_purpose":"It tracks the number of days until an Opportunity Closes","comma_separated_profiles":"Marketing User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/121","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_029","template_name":"add_formula_field_with_visibility","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/121"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/121"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/add_formula_field_with_visibility"}],"source_schema_notes":[],"template_registry":{"name":"add_formula_field_with_visibility","original":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn the Opportunity object, please add a formula field named “Days to close” that shows how many days are left until the Close Date. Lock down field visibility so only the Marketing User profile can see it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_001","query_templated":"Add a formula field with the name Days to close to the object Opportunity. It tracks the number of days until an Opportunity Closes. This newly added field should only be visible to Marketing User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn the Opportunity object, please add a formula field named “Days to close” that shows how many days are left until the Close Date. Lock down field visibility so only the Marketing User profile can see it.\n```","human_review_comment":"accept","ground_truth_dict":{"field_name":"Days_to_close__c","object_name":"Opportunity","formula":"CloseDate - TODAY()","comma_separated_profiles":"MarketingProfile"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Days to close","object_name":"Opportunity","field_purpose":"It tracks the number of days until an Opportunity Closes","comma_separated_profiles":"Marketing User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/121"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L17","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_add_formula_field_with_visibility","data_retrieval_spec":[{"type":"metadata","id":"object_fields","params":{"types_and_members":{"Profile":["*"],"CustomField":["{object_name}.{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d618223deaf4a4eac48fcb2479edaf159843527108de208aa72c1956f0831b14"} {"id":"scuba:admin_029_002","source_id":"scuba","original_id":"admin_029_002","kind":"benchmark_task","instruction":"For Accounts, create a formula field called “Check country” It should return true when the Billing Country matches the Account Owner’s Country, and false otherwise. Make this field visible only to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_002","query_templated":"Add a formula field with the name Check country to the object Account. It checks if the billing country of the account is the same as the account owner's country. This newly added field should only be visible to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.","query":"For Accounts, create a formula field called “Check country” It should return true when the Billing Country matches the Account Owner’s Country, and false otherwise. Make this field visible only to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.","human_review_comment":"delete .","ground_truth_dict":{"field_name":"Check_country__c","object_name":"Account","formula":"BillingCountry = Owner.Country","comma_separated_profiles":"MarketingProfile, Analytics Cloud Security User, Einstein Agent User"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Check country","object_name":"Account","field_purpose":"It checks if the billing country of the account is the same as the account owner's country","comma_separated_profiles":"Marketing User, Analytics Cloud Security User, and Einstein Agent User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/122","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_029","template_name":"add_formula_field_with_visibility","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/122"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/122"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/add_formula_field_with_visibility"}],"source_schema_notes":[],"template_registry":{"name":"add_formula_field_with_visibility","original":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nFor Accounts, create a formula field called “Check country” It should return true when the Billing Country matches the Account Owner’s Country, and false otherwise. Make this field visible only to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_002","query_templated":"Add a formula field with the name Check country to the object Account. It checks if the billing country of the account is the same as the account owner's country. This newly added field should only be visible to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nFor Accounts, create a formula field called “Check country” It should return true when the Billing Country matches the Account Owner’s Country, and false otherwise. Make this field visible only to Marketing User, Analytics Cloud Security User, and Einstein Agent User profiles.\n```","human_review_comment":"delete .","ground_truth_dict":{"field_name":"Check_country__c","object_name":"Account","formula":"BillingCountry = Owner.Country","comma_separated_profiles":"MarketingProfile, Analytics Cloud Security User, Einstein Agent User"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Check country","object_name":"Account","field_purpose":"It checks if the billing country of the account is the same as the account owner's country","comma_separated_profiles":"Marketing User, Analytics Cloud Security User, and Einstein Agent User"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/122"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L17","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_add_formula_field_with_visibility","data_retrieval_spec":[{"type":"metadata","id":"object_fields","params":{"types_and_members":{"Profile":["*"],"CustomField":["{object_name}.{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"71e4b1722512f955b9ac937d99d8b86ac792361a2fa95864dd32a094edd7801c"} {"id":"scuba:admin_029_003","source_id":"scuba","original_id":"admin_029_003","kind":"benchmark_task","instruction":"On the Lead object, set up a formula field named “Lead”. If the lead is still open, have it display the number of days it’s been open; if not, leave it blank. Limit visibility exclusively to the Everyone profiles.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_003","query_templated":"Add a formula field with the name Lead to the object Lead. It checks to see if a lead is open, and if so, calculates the number of days it has been open. This newly added field should only be visible to Everyone profiles.","query":"On the Lead object, set up a formula field named “Lead”. If the lead is still open, have it display the number of days it’s been open; if not, leave it blank. Limit visibility exclusively to the Everyone profiles.","human_review_comment":"delete .","ground_truth_dict":{"field_name":"Lead__c","object_name":"Lead","formula":"IF(ISPICKVAL(Status, \"Open\"), TODAY() - DATEVALUE(CreatedDate), NULL)","comma_separated_profiles":"Admin, Analytics Cloud Integration User, Analytics Cloud Security User, Anypoint Integration, Authenticated Website, B2B Reordering Portal Buyer Profile, ContractManager, Cross Org Data Proxy User, Custom%3A Marketing Profile, Custom%3A Sales Profile, Custom%3A Support Profile, Customer Community Login User, Customer Community Plus Login User, Customer Community Plus User, Customer Community User, Customer Portal Manager Custom, Customer Portal Manager Standard, Einstein Agent User, External Apps Login User, External Identity User, Force%2Ecom - App Subscription User, Force%2Ecom - Free User, Gold Partner User, High Volume Customer Portal User, HighVolumePortal, Identity User, MarketingProfile, Minimum Access - API Only Integrations, Minimum Access - Salesforce, Partner App Subscription User, Partner Community Login User, Partner Community User, PlatformPortal, Read Only, Salesforce API Only System Integrations, Silver Partner User, SolutionManager, Standard, StandardAul, Work%2Ecom Only User"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Lead","object_name":"Lead","field_purpose":"It checks to see if a lead is open, and if so, calculates the number of days it has been open","comma_separated_profiles":"Everyone"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/123","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_029","template_name":"add_formula_field_with_visibility","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/123"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/123"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/add_formula_field_with_visibility"}],"source_schema_notes":[],"template_registry":{"name":"add_formula_field_with_visibility","original":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn the Lead object, set up a formula field named “Lead”. If the lead is still open, have it display the number of days it’s been open; if not, leave it blank. Limit visibility exclusively to the Everyone profiles.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_003","query_templated":"Add a formula field with the name Lead to the object Lead. It checks to see if a lead is open, and if so, calculates the number of days it has been open. This newly added field should only be visible to Everyone profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn the Lead object, set up a formula field named “Lead”. If the lead is still open, have it display the number of days it’s been open; if not, leave it blank. Limit visibility exclusively to the Everyone profiles.\n```","human_review_comment":"delete .","ground_truth_dict":{"field_name":"Lead__c","object_name":"Lead","formula":"IF(ISPICKVAL(Status, \"Open\"), TODAY() - DATEVALUE(CreatedDate), NULL)","comma_separated_profiles":"Admin, Analytics Cloud Integration User, Analytics Cloud Security User, Anypoint Integration, Authenticated Website, B2B Reordering Portal Buyer Profile, ContractManager, Cross Org Data Proxy User, Custom%3A Marketing Profile, Custom%3A Sales Profile, Custom%3A Support Profile, Customer Community Login User, Customer Community Plus Login User, Customer Community Plus User, Customer Community User, Customer Portal Manager Custom, Customer Portal Manager Standard, Einstein Agent User, External Apps Login User, External Identity User, Force%2Ecom - App Subscription User, Force%2Ecom - Free User, Gold Partner User, High Volume Customer Portal User, HighVolumePortal, Identity User, MarketingProfile, Minimum Access - API Only Integrations, Minimum Access - Salesforce, Partner App Subscription User, Partner Community Login User, Partner Community User, PlatformPortal, Read Only, Salesforce API Only System Integrations, Silver Partner User, SolutionManager, Standard, StandardAul, Work%2Ecom Only User"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"Lead","object_name":"Lead","field_purpose":"It checks to see if a lead is open, and if so, calculates the number of days it has been open","comma_separated_profiles":"Everyone"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/123"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L17","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_add_formula_field_with_visibility","data_retrieval_spec":[{"type":"metadata","id":"object_fields","params":{"types_and_members":{"Profile":["*"],"CustomField":["{object_name}.{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c898301355997f4b22a654154dbd29c79dabab8f7b37650c27fdd30e62f9e751"} {"id":"scuba:admin_029_004","source_id":"scuba","original_id":"admin_029_004","kind":"benchmark_task","instruction":"Add a formula field “RevenueInK” on Account that takes Annual Revenue, divides it by 1,000, and appends “K” (e.g., 250,000 → 250K) to simplify reporting. Ensure only System Administrator, Custom: Sales, and Marketing User profiles can see it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_004","query_templated":"Add a formula field with the name RevenueInK to the object Account. Display the annual revenue divided by 1,000 and add 'K' (for easier reporting). This newly added field should only be visible to System Administrator, Custom: Sales, Marketing User profiles.","query":"Add a formula field “RevenueInK” on Account that takes Annual Revenue, divides it by 1,000, and appends “K” (e.g., 250,000 → 250K) to simplify reporting. Ensure only System Administrator, Custom: Sales, and Marketing User profiles can see it.","human_review_comment":"accept","ground_truth_dict":{"field_name":"RevenueInK__c","object_name":"Account","formula":"TEXT(AnnualRevenue /1000)&\"K\"","comma_separated_profiles":"Admin, Custom%3A Sales Profile, MarketingProfile"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"RevenueInK","object_name":"Account","field_purpose":"Display the annual revenue divided by 1,000 and add 'K' (for easier reporting)","comma_separated_profiles":"System Administrator, Custom: Sales, Marketing User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/124","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_029","template_name":"add_formula_field_with_visibility","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/124"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/124"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/add_formula_field_with_visibility"}],"source_schema_notes":[],"template_registry":{"name":"add_formula_field_with_visibility","original":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nAdd a formula field “RevenueInK” on Account that takes Annual Revenue, divides it by 1,000, and appends “K” (e.g., 250,000 → 250K) to simplify reporting. Ensure only System Administrator, Custom: Sales, and Marketing User profiles can see it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_004","query_templated":"Add a formula field with the name RevenueInK to the object Account. Display the annual revenue divided by 1,000 and add 'K' (for easier reporting). This newly added field should only be visible to System Administrator, Custom: Sales, Marketing User profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nAdd a formula field “RevenueInK” on Account that takes Annual Revenue, divides it by 1,000, and appends “K” (e.g., 250,000 → 250K) to simplify reporting. Ensure only System Administrator, Custom: Sales, and Marketing User profiles can see it.\n```","human_review_comment":"accept","ground_truth_dict":{"field_name":"RevenueInK__c","object_name":"Account","formula":"TEXT(AnnualRevenue /1000)&\"K\"","comma_separated_profiles":"Admin, Custom%3A Sales Profile, MarketingProfile"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"RevenueInK","object_name":"Account","field_purpose":"Display the annual revenue divided by 1,000 and add 'K' (for easier reporting)","comma_separated_profiles":"System Administrator, Custom: Sales, Marketing User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/124"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L17","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_add_formula_field_with_visibility","data_retrieval_spec":[{"type":"metadata","id":"object_fields","params":{"types_and_members":{"Profile":["*"],"CustomField":["{object_name}.{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f32bf1b5615822c2f7e4aa12d997befb623e35530644086601621548aa131d28"} {"id":"scuba:admin_029_005","source_id":"scuba","original_id":"admin_029_005","kind":"benchmark_task","instruction":"On Contact object, create a formula field called “IsVIP” that evaluates to true when the related Account’s Annual Revenue exceeds 1,000,000. Restrict field visibility to Standard User and Solution Manager profiles only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_005","query_templated":"Add a formula field with the name IsVIP to the object Contact. Is the Annual Revenue of the Account related to the contact greater than 1M?. This newly added field should only be visible to Standard User, Solution Manager profiles.","query":"On Contact object, create a formula field called “IsVIP” that evaluates to true when the related Account’s Annual Revenue exceeds 1,000,000. Restrict field visibility to Standard User and Solution Manager profiles only.","human_review_comment":"accept","ground_truth_dict":{"field_name":"IsVIP__c","object_name":"Contact","formula":"Account.AnnualRevenue > 1000000","comma_separated_profiles":"Standard, SolutionManager"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"IsVIP","object_name":"Contact","field_purpose":"Is the Annual Revenue of the Account related to the contact greater than 1M?","comma_separated_profiles":"Standard User, Solution Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/125","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_029","template_name":"add_formula_field_with_visibility","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/125"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/125"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/add_formula_field_with_visibility"}],"source_schema_notes":[],"template_registry":{"name":"add_formula_field_with_visibility","original":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/add_formula_field_with_visibility","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn Contact object, create a formula field called “IsVIP” that evaluates to true when the related Account’s Annual Revenue exceeds 1,000,000. Restrict field visibility to Standard User and Solution Manager profiles only.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_029_005","query_templated":"Add a formula field with the name IsVIP to the object Contact. Is the Annual Revenue of the Account related to the contact greater than 1M?. This newly added field should only be visible to Standard User, Solution Manager profiles.","query":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save.\n\n## Your ultimate task\n```\nOn Contact object, create a formula field called “IsVIP” that evaluates to true when the related Account’s Annual Revenue exceeds 1,000,000. Restrict field visibility to Standard User and Solution Manager profiles only.\n```","human_review_comment":"accept","ground_truth_dict":{"field_name":"IsVIP__c","object_name":"Contact","formula":"Account.AnnualRevenue > 1000000","comma_separated_profiles":"Standard, SolutionManager"},"difficulty":"hard","query_template_name":"add_formula_field_with_visibility","query_template_metadata":{"template_id":"admin_029","template_string":"Add a formula field with the name {field_name} to the object {object_name}. {field_purpose}. This newly added field should only be visible to {comma_separated_profiles} profiles.","prerequisites":{},"objects":[],"metadata_types":["CustomField"]},"instance_dict":{"field_name":"IsVIP","object_name":"Contact","field_purpose":"Is the Annual Revenue of the Account related to the contact greater than 1M?","comma_separated_profiles":"Standard User, Solution Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Add a formula field with the name [Days to close] to the object [Opportunity]. [It tracks the number of days until an Opportunity Closes]. This newly added field should only be visible to [Marketing User].\nEXPERIENCE:\nSuccessful plan to add the “Days to close” formula field on Opportunity and restrict visibility to Marketing User:\n\n- Navigate to Setup > Object Manager > Opportunity > Fields & Relationships.\n- Click New, select Formula, and click Next.\n- Set Field Label: Days to close; choose Return Type: Number; continue.\n- Build the formula:\n - Insert Field: Close Date\n - Operator: Subtract (-)\n - Function: TODAY()\n - Final formula: CloseDate - TODAY()\n- Proceed to Field-Level Security:\n - Deselect global visibility (uncheck “Visible to all”).\n - Select only the Marketing User profile for visibility.\n- Click Next, then Save.\nSample 2:\nTASK:Add a formula field with the name [Check country] to the object [Account] [It checks if the billing country of the account is the same as the account owner's country]. This newly added field should only be visible to [Marketing User, Analytics Cloud Security User, and Einstein Agent User].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate: Setup > Object Manager > Account > Fields & Relationships > New.\n- Choose field type: Formula. Set Return Type: Checkbox. Click Next.\n- Set Field Label: Check country (API Name auto-populates). Click Next.\n- Build the formula:\n - Insert Field: Billing Country.\n - Insert Operator: = (Equal).\n - Insert Field: Account Owner > Country.\n - This yields a boolean formula equivalent to: BillingCountry = Owner.Country.\n - Click Next.\n- Set Field-Level Security:\n - Deselect the “Visible” header checkbox to clear visibility for all profiles.\n - Check visibility only for:\n - Marketing User\n - Analytics Cloud Security User\n - Einstein Agent User\n - Click Next.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/125"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L17","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_add_formula_field_with_visibility","data_retrieval_spec":[{"type":"metadata","id":"object_fields","params":{"types_and_members":{"Profile":["*"],"CustomField":["{object_name}.{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a3137b2e10520e6964ec0022c3bbe6f3a6d3e3150a1d1c17fa667d7deb2817b5"} {"id":"scuba:admin_030_001","source_id":"scuba","original_id":"admin_030_001","kind":"benchmark_task","instruction":"On the Account object, add a validaiton rule: if the Account Number isn’t exactly 8 characters long, block the save and show this message: \"account number must be 8 characters long.\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_001","query_templated":"Create a validation rule for the Account Object. This rule checks if the Account Number field's characters length is not 8. If the rule is triggered, the error message should be account number must be 8 characters long..","query":"On the Account object, add a validaiton rule: if the Account Number isn’t exactly 8 characters long, block the save and show this message: \"account number must be 8 characters long.\".","human_review_comment":"tweaked","ground_truth_dict":{"object_name":"Account","error_condition_formula":"LEN(AccountNumber) != 8","error_message":"account number must be 8 characters long."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Account","error_condition_description":"This rule checks if the Account Number field's characters length is not 8","error_message":"account number must be 8 characters long."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/126","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_030","template_name":"create_validation_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/126"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/126"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_validation_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_validation_rule","original":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOn the Account object, add a validaiton rule: if the Account Number isn’t exactly 8 characters long, block the save and show this message: \"account number must be 8 characters long.\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_001","query_templated":"Create a validation rule for the Account Object. This rule checks if the Account Number field's characters length is not 8. If the rule is triggered, the error message should be account number must be 8 characters long..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOn the Account object, add a validaiton rule: if the Account Number isn’t exactly 8 characters long, block the save and show this message: \"account number must be 8 characters long.\".\n```","human_review_comment":"tweaked","ground_truth_dict":{"object_name":"Account","error_condition_formula":"LEN(AccountNumber) != 8","error_message":"account number must be 8 characters long."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Account","error_condition_description":"This rule checks if the Account Number field's characters length is not 8","error_message":"account number must be 8 characters long."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/126"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L54","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_validation_rule","data_retrieval_spec":[{"type":"tooling","id":"validation_rule_info","params":{"endpoint":"/services/data/v62.0/tooling/query?q=Select+ErrorMessage,Metadata,EntityDefinitionId,ValidationName+From+ValidationRule+WHERE+EntityDefinitionId='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3b0c70471a212e5b71d9d1fa42c4724f037ea679b6783b85c5aadb385ce90c1c"} {"id":"scuba:admin_030_002","source_id":"scuba","original_id":"admin_030_002","kind":"benchmark_task","instruction":"For the Contacts obeject, enforce corporate emails only—create a validation that fails when the Email field does not contain \"@company.com\", and display: \"Email must be a company email ending in '@company.com'.\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_002","query_templated":"Create a validation rule for the Contact Object. This rule should check if the Email field does not contain \"@company.com\". If the rule is triggered, the error message should be Email must be a company email ending in '@company.com'..","query":"For the Contacts obeject, enforce corporate emails only—create a validation that fails when the Email field does not contain \"@company.com\", and display: \"Email must be a company email ending in '@company.com'.\".","human_review_comment":"tweaked","ground_truth_dict":{"object_name":"Contact","error_condition_formula":"NOT(CONTAINS(Email, \"@company.com\"))","error_message":"Email must be a company email ending in '@company.com'."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Contact","error_condition_description":"This rule should check if the Email field does not contain \"@company.com\"","error_message":"Email must be a company email ending in '@company.com'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/127","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_030","template_name":"create_validation_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/127"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/127"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_validation_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_validation_rule","original":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor the Contacts obeject, enforce corporate emails only—create a validation that fails when the Email field does not contain \"@company.com\", and display: \"Email must be a company email ending in '@company.com'.\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_002","query_templated":"Create a validation rule for the Contact Object. This rule should check if the Email field does not contain \"@company.com\". If the rule is triggered, the error message should be Email must be a company email ending in '@company.com'..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor the Contacts obeject, enforce corporate emails only—create a validation that fails when the Email field does not contain \"@company.com\", and display: \"Email must be a company email ending in '@company.com'.\".\n```","human_review_comment":"tweaked","ground_truth_dict":{"object_name":"Contact","error_condition_formula":"NOT(CONTAINS(Email, \"@company.com\"))","error_message":"Email must be a company email ending in '@company.com'."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Contact","error_condition_description":"This rule should check if the Email field does not contain \"@company.com\"","error_message":"Email must be a company email ending in '@company.com'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/127"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L54","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_validation_rule","data_retrieval_spec":[{"type":"tooling","id":"validation_rule_info","params":{"endpoint":"/services/data/v62.0/tooling/query?q=Select+ErrorMessage,Metadata,EntityDefinitionId,ValidationName+From+ValidationRule+WHERE+EntityDefinitionId='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"ec93cc0bb9e88fc5798d3453567cf85fd941bd9ea493e39a4b4ef241cdae58f1"} {"id":"scuba:admin_030_003","source_id":"scuba","original_id":"admin_030_003","kind":"benchmark_task","instruction":"Create a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be \"Close Date cannot be in the past.\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_003","query_templated":"Create a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be Close Date cannot be in the past..","query":"Create a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be \"Close Date cannot be in the past.\".","human_review_comment":"rejected; use the original","ground_truth_dict":{"object_name":"Opportunity","error_condition_formula":"CloseDate < TODAY()","error_message":"Close Date cannot be in the past."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Opportunity","error_condition_description":"This rule should trigger an error if the Close Date is set to a date in the past","error_message":"Close Date cannot be in the past."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/128","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_030","template_name":"create_validation_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/128"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/128"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_validation_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_validation_rule","original":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be \"Close Date cannot be in the past.\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_003","query_templated":"Create a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be Close Date cannot be in the past..","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a validation rule for the Opportunity Object. This rule should trigger an error if the Close Date is set to a date in the past. If the rule is triggered, the error message should be \"Close Date cannot be in the past.\".\n```","human_review_comment":"rejected; use the original","ground_truth_dict":{"object_name":"Opportunity","error_condition_formula":"CloseDate < TODAY()","error_message":"Close Date cannot be in the past."},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Opportunity","error_condition_description":"This rule should trigger an error if the Close Date is set to a date in the past","error_message":"Close Date cannot be in the past."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/128"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L54","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_validation_rule","data_retrieval_spec":[{"type":"tooling","id":"validation_rule_info","params":{"endpoint":"/services/data/v62.0/tooling/query?q=Select+ErrorMessage,Metadata,EntityDefinitionId,ValidationName+From+ValidationRule+WHERE+EntityDefinitionId='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"adf91fd8d7401184ebb6228cab36d6372d8e4e7bc2555c07e17bc91ba5c37a63"} {"id":"scuba:admin_030_004","source_id":"scuba","original_id":"admin_030_004","kind":"benchmark_task","instruction":"On the Leads, I want to a new validation rule. Disallow zero or negative values for Annual Revenue. If a user enters anything ≤ 0, trigger the error with a message \"Annual Revenue must be greater than 0\".\r\n","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_004","query_templated":"Create a validation rule for the Lead Object. The Annual Revenue should be greater than 0. If the rule is triggered, the error message should be Annual Revenue must be greater than 0.","query":"On the Leads, I want to a new validation rule. Disallow zero or negative values for Annual Revenue. If a user enters anything ≤ 0, trigger the error with a message \"Annual Revenue must be greater than 0\".\r\n","human_review_comment":"rewritte","ground_truth_dict":{"object_name":"Lead","error_condition_formula":"AnnualRevenue <= 0","error_message":"Annual Revenue must be greater than 0"},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Lead","error_condition_description":"The Annual Revenue should be greater than 0","error_message":"Annual Revenue must be greater than 0"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/129","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_030","template_name":"create_validation_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/129"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/129"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_validation_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_validation_rule","original":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOn the Leads, I want to a new validation rule. Disallow zero or negative values for Annual Revenue. If a user enters anything ≤ 0, trigger the error with a message \"Annual Revenue must be greater than 0\".\r\n\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_004","query_templated":"Create a validation rule for the Lead Object. The Annual Revenue should be greater than 0. If the rule is triggered, the error message should be Annual Revenue must be greater than 0.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOn the Leads, I want to a new validation rule. Disallow zero or negative values for Annual Revenue. If a user enters anything ≤ 0, trigger the error with a message \"Annual Revenue must be greater than 0\".\r\n\n```","human_review_comment":"rewritte","ground_truth_dict":{"object_name":"Lead","error_condition_formula":"AnnualRevenue <= 0","error_message":"Annual Revenue must be greater than 0"},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Lead","error_condition_description":"The Annual Revenue should be greater than 0","error_message":"Annual Revenue must be greater than 0"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/129"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L54","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_validation_rule","data_retrieval_spec":[{"type":"tooling","id":"validation_rule_info","params":{"endpoint":"/services/data/v62.0/tooling/query?q=Select+ErrorMessage,Metadata,EntityDefinitionId,ValidationName+From+ValidationRule+WHERE+EntityDefinitionId='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"459bf6ed9edd8cc3d7b355102da26bd36a843e997de644fa9dea4a958a3fd2b5"} {"id":"scuba:admin_030_005","source_id":"scuba","original_id":"admin_030_005","kind":"benchmark_task","instruction":"For the Assets object, build a validation rule for me. Require the Description field to have a minimum of 20 characters only when the Asset Owner’s Email contains \"example.com\"; if it’s shorter, show: \"Description must be at least 20 characters for example.com users\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_005","query_templated":"Create a validation rule for the Asset Object. The rule is that the Description field must contain at least 20 characters if the Asset Owner's Email field contains 'example.com'. If the rule is triggered, the error message should be Description must be at least 20 characters for example.com users.","query":"For the Assets object, build a validation rule for me. Require the Description field to have a minimum of 20 characters only when the Asset Owner’s Email contains \"example.com\"; if it’s shorter, show: \"Description must be at least 20 characters for example.com users\".","human_review_comment":"rewritte; update GT dict.","ground_truth_dict":{"object_name":"Asset","error_condition_formula":"IF(CONTAINS(Owner.Email, \"example.com\"), LEN(Description)<20, false)","error_message":"Description must be at least 20 characters for example.com users"},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Asset","error_condition_description":"The rule is that the Description field must contain at least 20 characters if the Asset Owner's Email field contains 'example.com'","error_message":"Description must be at least 20 characters for example.com users"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/130","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Formulas and Validations","source_difficulty":"hard","template_id":"admin_030","template_name":"create_validation_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/130"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/130"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_validation_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_validation_rule","original":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_validation_rule","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor the Assets object, build a validation rule for me. Require the Description field to have a minimum of 20 characters only when the Asset Owner’s Email contains \"example.com\"; if it’s shorter, show: \"Description must be at least 20 characters for example.com users\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Formulas and Validations","task_id":"admin_030_005","query_templated":"Create a validation rule for the Asset Object. The rule is that the Description field must contain at least 20 characters if the Asset Owner's Email field contains 'example.com'. If the rule is triggered, the error message should be Description must be at least 20 characters for example.com users.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor the Assets object, build a validation rule for me. Require the Description field to have a minimum of 20 characters only when the Asset Owner’s Email contains \"example.com\"; if it’s shorter, show: \"Description must be at least 20 characters for example.com users\".\n```","human_review_comment":"rewritte; update GT dict.","ground_truth_dict":{"object_name":"Asset","error_condition_formula":"IF(CONTAINS(Owner.Email, \"example.com\"), LEN(Description)<20, false)","error_message":"Description must be at least 20 characters for example.com users"},"difficulty":"hard","query_template_name":"create_validation_rule","query_template_metadata":{"template_id":"admin_030","template_string":"Create a validation rule for the {object_name} Object. {error_condition_description}. If the rule is triggered, the error message should be {error_message}.","prerequisites":{},"objects":[],"metadata_types":["ValidationRule"]},"instance_dict":{"object_name":"Asset","error_condition_description":"The rule is that the Description field must contain at least 20 characters if the Asset Owner's Email field contains 'example.com'","error_message":"Description must be at least 20 characters for example.com users"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a validation rule for the [Account] Object. [This rule checks if the Account Number field's characters length is not 8]. If the rule is triggered, the error message should be [account number must be 8 characters long.] \nEXPERIENCE:\nSuccessful plan to create the validation rule on Account:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for and open Object Manager.\n- Open the Account object.\n- Go to Validation Rules and click New.\n- Set Rule Name: AccountNumber_Length_Check.\n- Set Error Condition Formula: LEN(AccountNumber) <> 8\n- Set Error Message: account number must be 8 characters long.\n- Click Check Syntax, then Save.\nSample 2:\nTASK:Create a validation rule for the Contact object. This rule should check if the Email field does not contain \"@company.com\". If the email address is from a different domain, the error message should be: \"Email must be a company email ending in '@company.com'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Setup (gear icon > Setup).\n- In Setup, navigate: Objects and Fields > Object Manager.\n- Use Quick Find to search for “Contact” and open the Contact object.\n- Go to Validation Rules and click New.\n- Set Rule Name: Enforce_Company_Email_Domain.\n- Set Error Condition Formula: NOT(CONTAINS(Email, \"@company.com\"))\n- Set Error Message: Email must be a company email ending in '@company.com'.\n- Click Check Syntax, then Save.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/130"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L54","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_validation_rule","data_retrieval_spec":[{"type":"tooling","id":"validation_rule_info","params":{"endpoint":"/services/data/v62.0/tooling/query?q=Select+ErrorMessage,Metadata,EntityDefinitionId,ValidationName+From+ValidationRule+WHERE+EntityDefinitionId='{object_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a9c703893d05403091defd0500671dde1461a319ac5bc8a0309709072d99c6b5"} {"id":"scuba:admin_031_001","source_id":"scuba","original_id":"admin_031_001","kind":"benchmark_task","instruction":"In the Campaign object, move the Status picklist so that \"Aborted\" sits at the very top and make it the default selection for new Campaigns.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_001","query_templated":"Reorder the picklist values of Campaign Object's Status field. by moving the value \"Aborted\" to the top and make it the default value","query":"In the Campaign object, move the Status picklist so that \"Aborted\" sits at the very top and make it the default selection for new Campaigns.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Campaign","field_name":"Status","comma_separated_values":"Aborted, Planned, In Progress, Completed","default_value":"Aborted"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Reorder the picklist values of","object_name":"Campaign","field_name":"Status","additional_instructions":"by moving the value \"Aborted\" to the top and make it the default value"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/131","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"medium","template_id":"admin_031","template_name":"picklist_administration_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/131"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/131"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/picklist_administration_actions"}],"source_schema_notes":[],"template_registry":{"name":"picklist_administration_actions","original":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nIn the Campaign object, move the Status picklist so that \"Aborted\" sits at the very top and make it the default selection for new Campaigns.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_001","query_templated":"Reorder the picklist values of Campaign Object's Status field. by moving the value \"Aborted\" to the top and make it the default value","query":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nIn the Campaign object, move the Status picklist so that \"Aborted\" sits at the very top and make it the default selection for new Campaigns.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Campaign","field_name":"Status","comma_separated_values":"Aborted, Planned, In Progress, Completed","default_value":"Aborted"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Reorder the picklist values of","object_name":"Campaign","field_name":"Status","additional_instructions":"by moving the value \"Aborted\" to the top and make it the default value"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/131"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L80","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_picklist_administration_actions","data_retrieval_spec":[{"type":"metadata","id":"picklist_info","params":{"types_and_members":{"StandardValueSet":["{object_name}{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"873772915b290e284987c7f3078e3073fccb7b1c4857fe2903d6518fe7ba498e"} {"id":"scuba:admin_031_002","source_id":"scuba","original_id":"admin_031_002","kind":"benchmark_task","instruction":"On Cases, please add a new Status option called \"Pending\" to the picklist.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_002","query_templated":"Create a new picklist value \"Pending\" in the Case Object's Status field. ","query":"On Cases, please add a new Status option called \"Pending\" to the picklist.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","field_name":"Status","comma_separated_values":"New, Working, Escalated, Closed, Pending","default_value":"New"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Create a new picklist value \"Pending\" in the","object_name":"Case","field_name":"Status","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/132","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"medium","template_id":"admin_031","template_name":"picklist_administration_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/132"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/132"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/picklist_administration_actions"}],"source_schema_notes":[],"template_registry":{"name":"picklist_administration_actions","original":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nOn Cases, please add a new Status option called \"Pending\" to the picklist.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_002","query_templated":"Create a new picklist value \"Pending\" in the Case Object's Status field. ","query":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nOn Cases, please add a new Status option called \"Pending\" to the picklist.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","field_name":"Status","comma_separated_values":"New, Working, Escalated, Closed, Pending","default_value":"New"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Create a new picklist value \"Pending\" in the","object_name":"Case","field_name":"Status","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/132"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L80","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_picklist_administration_actions","data_retrieval_spec":[{"type":"metadata","id":"picklist_info","params":{"types_and_members":{"StandardValueSet":["{object_name}{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"89b4e61495be380579727a39279d2a3ef485fa60aa4cd3293d94584dc2fb51da"} {"id":"scuba:admin_031_003","source_id":"scuba","original_id":"admin_031_003","kind":"benchmark_task","instruction":"Update the Lead Status picklist by renaming the value \"Closed - Not Converted\" to \"Closed - Incomplete.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_003","query_templated":"Rename the picklist value from \"Closed - Not Converted\" to \"Closed - Incomplete\" for the Lead Object's Lead Status field. ","query":"Update the Lead Status picklist by renaming the value \"Closed - Not Converted\" to \"Closed - Incomplete.\"","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","field_name":"Status","comma_separated_values":"Open - Not Contacted, Working - Contacted, Closed - Converted, Closed - Incomplete","default_value":"Open - Not Contacted"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Rename the picklist value from \"Closed - Not Converted\" to \"Closed - Incomplete\" for the","object_name":"Lead","field_name":"Lead Status","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/133","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"medium","template_id":"admin_031","template_name":"picklist_administration_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/133"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/133"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/picklist_administration_actions"}],"source_schema_notes":[],"template_registry":{"name":"picklist_administration_actions","original":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nUpdate the Lead Status picklist by renaming the value \"Closed - Not Converted\" to \"Closed - Incomplete.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_003","query_templated":"Rename the picklist value from \"Closed - Not Converted\" to \"Closed - Incomplete\" for the Lead Object's Lead Status field. ","query":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nUpdate the Lead Status picklist by renaming the value \"Closed - Not Converted\" to \"Closed - Incomplete.\"\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","field_name":"Status","comma_separated_values":"Open - Not Contacted, Working - Contacted, Closed - Converted, Closed - Incomplete","default_value":"Open - Not Contacted"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Rename the picklist value from \"Closed - Not Converted\" to \"Closed - Incomplete\" for the","object_name":"Lead","field_name":"Lead Status","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/133"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L80","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_picklist_administration_actions","data_retrieval_spec":[{"type":"metadata","id":"picklist_info","params":{"types_and_members":{"StandardValueSet":["{object_name}{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6dd16729212b309d8ba9f64bd0d8186b454354bd0b1d15b21a8265259999eb11"} {"id":"scuba:admin_031_004","source_id":"scuba","original_id":"admin_031_004","kind":"benchmark_task","instruction":"Add \"Social Media\" as a new choice in the LeadSource picklist on the Lead object.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_004","query_templated":"Add a new picklist value \"Social Media\" to the Lead Object's LeadSource field. ","query":"Add \"Social Media\" as a new choice in the LeadSource picklist on the Lead object.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","field_name":"Source","comma_separated_values":"Web, Phone Inquiry, Partner Referral, Purchased List, Other, Social Media","default_value":""},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Add a new picklist value \"Social Media\" to the","object_name":"Lead","field_name":"LeadSource","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/134","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"medium","template_id":"admin_031","template_name":"picklist_administration_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/134"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/134"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/picklist_administration_actions"}],"source_schema_notes":[],"template_registry":{"name":"picklist_administration_actions","original":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nAdd \"Social Media\" as a new choice in the LeadSource picklist on the Lead object.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_004","query_templated":"Add a new picklist value \"Social Media\" to the Lead Object's LeadSource field. ","query":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nAdd \"Social Media\" as a new choice in the LeadSource picklist on the Lead object.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Lead","field_name":"Source","comma_separated_values":"Web, Phone Inquiry, Partner Referral, Purchased List, Other, Social Media","default_value":""},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Add a new picklist value \"Social Media\" to the","object_name":"Lead","field_name":"LeadSource","additional_instructions":""},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/134"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L80","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_picklist_administration_actions","data_retrieval_spec":[{"type":"metadata","id":"picklist_info","params":{"types_and_members":{"StandardValueSet":["{object_name}{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"7f32110337100ff234b628a11501a4380f561837b19aa8e94e6c8095aac7e799"} {"id":"scuba:admin_031_005","source_id":"scuba","original_id":"admin_031_005","kind":"benchmark_task","instruction":"For the Case object, rearrange the Priority picklist so \"Low\" is the first item and set it as the default value.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_005","query_templated":"Reorder the picklist values of the Case Object's Priority field. Move 'Low' to the top and make it the default value","query":"For the Case object, rearrange the Priority picklist so \"Low\" is the first item and set it as the default value.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","field_name":"Priority","comma_separated_values":"Low, High, Medium","default_value":"Low"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Reorder the picklist values of the","object_name":"Case","field_name":"Priority","additional_instructions":"Move 'Low' to the top and make it the default value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/135","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"medium","template_id":"admin_031","template_name":"picklist_administration_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/135"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/135"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/picklist_administration_actions"}],"source_schema_notes":[],"template_registry":{"name":"picklist_administration_actions","original":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/picklist_administration_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nFor the Case object, rearrange the Priority picklist so \"Low\" is the first item and set it as the default value.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_031_005","query_templated":"Reorder the picklist values of the Case Object's Priority field. Move 'Low' to the top and make it the default value","query":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save.\n\n## Your ultimate task\n```\nFor the Case object, rearrange the Priority picklist so \"Low\" is the first item and set it as the default value.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","field_name":"Priority","comma_separated_values":"Low, High, Medium","default_value":"Low"},"difficulty":"medium","query_template_name":"picklist_administration_actions","query_template_metadata":{"template_id":"admin_031","template_string":"{action_description} {object_name} Object's {field_name} field. {additional_instructions}","prerequisites":{},"objects":[],"metadata_types":["StandardValueSet"]},"instance_dict":{"action_description":"Reorder the picklist values of the","object_name":"Case","field_name":"Priority","additional_instructions":"Move 'Low' to the top and make it the default value"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Reorder the picklist values of] the [Campaign]'s [Status] field [by moving the value \"Aborted\" to the top and make it the default value].\nEXPERIENCE:\nSuccessful plan to reorder Campaign Status picklist and set default to Aborted:\n\n- Log in and open Setup.\n- Navigate to Object Manager.\n- Use Quick Find to locate and open the Campaign object.\n- Go to Fields & Relationships, search for and open the Status field.\n- Click Reorder, select “Aborted,” use “Move to Top,” then Save.\n- Click Edit on the picklist values screen, check the option to make “Aborted” the default, then Save.\nSample 2:\nTASK:Create a new picklist value \"Pending\" in the Case object's Status field.\nEXPERIENCE:\nSuccessful plan to create a new picklist value \"Pending\" in Case Status:\n\n1. Log in to Salesforce.\n2. Open Setup (gear icon > Setup).\n3. In Setup, navigate to Objects and Fields > Object Manager.\n4. Open the Case object.\n5. Go to Fields & Relationships.\n6. Select the Status field.\n7. Click New Picklist Value.\n8. Enter Pending as the value.\n9. Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/135"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L80","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_picklist_administration_actions","data_retrieval_spec":[{"type":"metadata","id":"picklist_info","params":{"types_and_members":{"StandardValueSet":["{object_name}{field_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"758079555be4b6315166d2958df3d95206dbb2f3db2044a3f5d55a3db2a74b9c"} {"id":"scuba:admin_032_001","source_id":"scuba","original_id":"admin_032_001","kind":"benchmark_task","instruction":"Please add a global value set called Ingredients and include these picklist options: sugar, butter, and milk. Make sure the values are displayed A–Z.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_001","query_templated":"Create a global value set with the name Ingredients and it should have sugar, butter, milk as the pick list values. Sort the values alphabetically.","query":"Please add a global value set called Ingredients and include these picklist options: sugar, butter, and milk. Make sure the values are displayed A–Z.","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"Ingredients","comma_separated_values":"sugar, butter, milk"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"Ingredients","comma_separated_values":"sugar, butter, milk"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/136","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"easy","template_id":"admin_032","template_name":"create_global_value_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/136"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/136"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_global_value_set"}],"source_schema_notes":[],"template_registry":{"name":"create_global_value_set","original":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nPlease add a global value set called Ingredients and include these picklist options: sugar, butter, and milk. Make sure the values are displayed A–Z.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_001","query_templated":"Create a global value set with the name Ingredients and it should have sugar, butter, milk as the pick list values. Sort the values alphabetically.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nPlease add a global value set called Ingredients and include these picklist options: sugar, butter, and milk. Make sure the values are displayed A–Z.\n```","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"Ingredients","comma_separated_values":"sugar, butter, milk"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"Ingredients","comma_separated_values":"sugar, butter, milk"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/136"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L104","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_global_value_set","data_retrieval_spec":[{"type":"metadata","id":"global_value_set_info","params":{"types_and_members":{"GlobalValueSet":["{value_set_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d72e94f2ec3fee9cc2b78a7c7aea1a4e6eb16e53e4154371b758773876ebd665"} {"id":"scuba:admin_032_002","source_id":"scuba","original_id":"admin_032_002","kind":"benchmark_task","instruction":"Could you set up a global value set named annotators with the picklist values michael, fabri, and caitlyn, and have them appear in alphabetical order?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_002","query_templated":"Create a global value set with the name annotators and it should have michael, fabri, caitlyn as the pick list values. Sort the values alphabetically.","query":"Could you set up a global value set named annotators with the picklist values michael, fabri, and caitlyn, and have them appear in alphabetical order?","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"annotators","comma_separated_values":"michael, fabri, caitlyn"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"annotators","comma_separated_values":"michael, fabri, caitlyn"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/137","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"easy","template_id":"admin_032","template_name":"create_global_value_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/137"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/137"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_global_value_set"}],"source_schema_notes":[],"template_registry":{"name":"create_global_value_set","original":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nCould you set up a global value set named annotators with the picklist values michael, fabri, and caitlyn, and have them appear in alphabetical order?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_002","query_templated":"Create a global value set with the name annotators and it should have michael, fabri, caitlyn as the pick list values. Sort the values alphabetically.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nCould you set up a global value set named annotators with the picklist values michael, fabri, and caitlyn, and have them appear in alphabetical order?\n```","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"annotators","comma_separated_values":"michael, fabri, caitlyn"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"annotators","comma_separated_values":"michael, fabri, caitlyn"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/137"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L104","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_global_value_set","data_retrieval_spec":[{"type":"metadata","id":"global_value_set_info","params":{"types_and_members":{"GlobalValueSet":["{value_set_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"386dbb2419c9b5b706b66971cb9089bb82fe087beddfeebd61a203fe3532cd69"} {"id":"scuba:admin_032_003","source_id":"scuba","original_id":"admin_032_003","kind":"benchmark_task","instruction":"Provision a global value set titled projects. Populate it with webagent, mm-rag, and human vs. syntheic, then alphabetize the picklist.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_003","query_templated":"Create a global value set with the name projects and it should have webagent, mm-rag, human vs. syntheic as the pick list values. Sort the values alphabetically.","query":"Provision a global value set titled projects. Populate it with webagent, mm-rag, and human vs. syntheic, then alphabetize the picklist.","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"projects","comma_separated_values":"webagent, mm-rag, human vs. syntheic"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"projects","comma_separated_values":"webagent, mm-rag, human vs. syntheic"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/138","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"easy","template_id":"admin_032","template_name":"create_global_value_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/138"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/138"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_global_value_set"}],"source_schema_notes":[],"template_registry":{"name":"create_global_value_set","original":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nProvision a global value set titled projects. Populate it with webagent, mm-rag, and human vs. syntheic, then alphabetize the picklist.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_003","query_templated":"Create a global value set with the name projects and it should have webagent, mm-rag, human vs. syntheic as the pick list values. Sort the values alphabetically.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nProvision a global value set titled projects. Populate it with webagent, mm-rag, and human vs. syntheic, then alphabetize the picklist.\n```","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"projects","comma_separated_values":"webagent, mm-rag, human vs. syntheic"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"projects","comma_separated_values":"webagent, mm-rag, human vs. syntheic"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/138"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L104","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_global_value_set","data_retrieval_spec":[{"type":"metadata","id":"global_value_set_info","params":{"types_and_members":{"GlobalValueSet":["{value_set_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f288fbc7a5a240b6be40334dab3a8f7cae5b94767df3ba3619d73497fb7f8dd9"} {"id":"scuba:admin_032_004","source_id":"scuba","original_id":"admin_032_004","kind":"benchmark_task","instruction":"I’m organizing a global value set for RocketLaunchPhases—load it with these entries: Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing. Sort the values alphabetically.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_004","query_templated":"Create a global value set with the name RocketLaunchPhases and it should have Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing as the pick list values. Sort the values alphabetically.","query":"I’m organizing a global value set for RocketLaunchPhases—load it with these entries: Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing. Sort the values alphabetically.","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"RocketLaunchPhases","comma_separated_values":"Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"RocketLaunchPhases","comma_separated_values":"Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/139","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"easy","template_id":"admin_032","template_name":"create_global_value_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/139"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/139"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_global_value_set"}],"source_schema_notes":[],"template_registry":{"name":"create_global_value_set","original":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nI’m organizing a global value set for RocketLaunchPhases—load it with these entries: Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing. Sort the values alphabetically.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_004","query_templated":"Create a global value set with the name RocketLaunchPhases and it should have Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing as the pick list values. Sort the values alphabetically.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nI’m organizing a global value set for RocketLaunchPhases—load it with these entries: Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing. Sort the values alphabetically.\n```","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"RocketLaunchPhases","comma_separated_values":"Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"RocketLaunchPhases","comma_separated_values":"Pre-Launch Checks, Countdown, Ignition, Lift-Off, Max Q, Stage Separation, Orbit Insertion, Payload Deployment, Re-Entry, Landing"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/139"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L104","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_global_value_set","data_retrieval_spec":[{"type":"metadata","id":"global_value_set_info","params":{"types_and_members":{"GlobalValueSet":["{value_set_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"621eb9f8cc0d0e2c4c3587559be830d8e3b7bb924a440fefdb95e108de5d4a82"} {"id":"scuba:admin_032_005","source_id":"scuba","original_id":"admin_032_005","kind":"benchmark_task","instruction":"Create a global value set named CustomerSatisfactionLevel and add the following choices: Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable. Order them A to Z.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_005","query_templated":"Create a global value set with the name CustomerSatisfactionLevel and it should have Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable as the pick list values. Sort the values alphabetically.","query":"Create a global value set named CustomerSatisfactionLevel and add the following choices: Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable. Order them A to Z.","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"CustomerSatisfactionLevel","comma_separated_values":"Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"CustomerSatisfactionLevel","comma_separated_values":"Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/140","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Picklist Administration","source_difficulty":"easy","template_id":"admin_032","template_name":"create_global_value_set","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/140"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/140"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_global_value_set"}],"source_schema_notes":[],"template_registry":{"name":"create_global_value_set","original":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_global_value_set","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nCreate a global value set named CustomerSatisfactionLevel and add the following choices: Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable. Order them A to Z.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Picklist Administration","task_id":"admin_032_005","query_templated":"Create a global value set with the name CustomerSatisfactionLevel and it should have Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable as the pick list values. Sort the values alphabetically.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”.\n\n## Your ultimate task\n```\nCreate a global value set named CustomerSatisfactionLevel and add the following choices: Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable. Order them A to Z.\n```","human_review_comment":"accept","ground_truth_dict":{"value_set_name":"CustomerSatisfactionLevel","comma_separated_values":"Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable"},"difficulty":"easy","query_template_name":"create_global_value_set","query_template_metadata":{"template_id":"admin_032","template_string":"Create a global value set with the name {value_set_name} and it should have {comma_separated_values} as the pick list values. Sort the values alphabetically.","prerequisites":{},"objects":[],"metadata_types":["GlobalValueSet"]},"instance_dict":{"value_set_name":"CustomerSatisfactionLevel","comma_separated_values":"Very Satisfied, Satisfied, Neutral, Unsatisfied, Very Unsatisfied, Not Applicable"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a global value set with the name [Ingredients] and it should have [sugar, butter, and milk] as the pick list values. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global value set \"Ingredients\" with alphabetically sorted values:\n\n- Log in to Salesforce and go to Setup.\n- In Quick Find, type “picklist” and select “Picklist Value Sets.”\n- Click “New” to create a new global value set.\n- Enter Label: “Ingredients” (API Name will auto-populate when focusing the Name field).\n- In the Values area, add each value on its own line:\n - sugar\n - butter\n - milk\n- Enable the option to display/sort values alphabetically.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\n- The alphabetical sort ensures the picklist values appear as: butter, milk, sugar.\nSample 2:\nTASK:Create a global value set with the name [annotators] and it should have [michael, fabri, and caitlyn] as the pick list values. Set the description to [annotators for the webagent task]. Sort the values alphabetically.\nEXPERIENCE:\nSuccessful plan to create the global picklist value set:\n\n- Log in to Salesforce and open Setup.\n- In the Setup Quick Find box, search for “picklist” and select “Picklist Value Sets” (Global Value Sets).\n- Click “New” to create a new value set.\n- Set Label to “annotators” (API Name will auto-populate).\n- In the Values box, enter each on a new line:\n - michael\n - fabri\n - caitlyn\n- Set Description to “annotators for the webagent task”.\n- Enable “Sort values alphabetically”.\n- Click “Save”."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/140"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L104","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_global_value_set","data_retrieval_spec":[{"type":"metadata","id":"global_value_set_info","params":{"types_and_members":{"GlobalValueSet":["{value_set_name}"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"458fab5a273fd7a36989a766fcac4be156534ec8d59885dadf20f55498bd00d4"} {"id":"scuba:admin_033_001","source_id":"scuba","original_id":"admin_033_001","kind":"benchmark_task","instruction":"Please tune the org-wide password settings so that passwords expire every 180 days, and make lockouts permanent.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_001","query_templated":"Update the org's password policies as follows: update the password expire days to 180 days and set lockout effective period to forever.","query":"Please tune the org-wide password settings so that passwords expire every 180 days, and make lockouts permanent.","human_review_comment":"rewrite","ground_truth_dict":{"expiration":"SixMonths","lockoutInterval":"Forever"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"update the password expire days to 180 days and set lockout effective period to forever"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/141","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Password Policies","source_difficulty":"easy","template_id":"admin_033","template_name":"update_password_policies","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/141"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/141"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/update_password_policies"}],"source_schema_notes":[],"template_registry":{"name":"update_password_policies","original":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease tune the org-wide password settings so that passwords expire every 180 days, and make lockouts permanent.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_001","query_templated":"Update the org's password policies as follows: update the password expire days to 180 days and set lockout effective period to forever.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease tune the org-wide password settings so that passwords expire every 180 days, and make lockouts permanent.\n```","human_review_comment":"rewrite","ground_truth_dict":{"expiration":"SixMonths","lockoutInterval":"Forever"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"update the password expire days to 180 days and set lockout effective period to forever"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/141"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L139","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_password_policies","data_retrieval_spec":[{"type":"metadata","id":"password_policy_info","params":{"types_and_members":{"Settings":["Security"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"85881cac4b957141f112b46c4bfafc016002fab767e79af293b11e077e3951b6"} {"id":"scuba:admin_033_002","source_id":"scuba","original_id":"admin_033_002","kind":"benchmark_task","instruction":"Tighten our password policy: require passwords to include at least one number plus both uppercase and lowercase letters, and cap failed login attempts at 5 before lockout.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_002","query_templated":"Update the org's password policies as follows: update the Password complexity requirements to Must include numbers and uppercase and lowercase letters, and Maximum invalid login attempts to 5.","query":"Tighten our password policy: require passwords to include at least one number plus both uppercase and lowercase letters, and cap failed login attempts at 5 before lockout.","human_review_comment":"accept","ground_truth_dict":{"complexity":"UpperLowerCaseNumeric","maxLoginAttempts":"FiveAttempts"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"update the Password complexity requirements to Must include numbers and uppercase and lowercase letters, and Maximum invalid login attempts to 5"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/142","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Password Policies","source_difficulty":"easy","template_id":"admin_033","template_name":"update_password_policies","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/142"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/142"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/update_password_policies"}],"source_schema_notes":[],"template_registry":{"name":"update_password_policies","original":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nTighten our password policy: require passwords to include at least one number plus both uppercase and lowercase letters, and cap failed login attempts at 5 before lockout.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_002","query_templated":"Update the org's password policies as follows: update the Password complexity requirements to Must include numbers and uppercase and lowercase letters, and Maximum invalid login attempts to 5.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nTighten our password policy: require passwords to include at least one number plus both uppercase and lowercase letters, and cap failed login attempts at 5 before lockout.\n```","human_review_comment":"accept","ground_truth_dict":{"complexity":"UpperLowerCaseNumeric","maxLoginAttempts":"FiveAttempts"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"update the Password complexity requirements to Must include numbers and uppercase and lowercase letters, and Maximum invalid login attempts to 5"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/142"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L139","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_password_policies","data_retrieval_spec":[{"type":"metadata","id":"password_policy_info","params":{"types_and_members":{"Settings":["Security"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"505f319cc4dc3c4b5ab38981495e51f45aa35a8930906711e1c50bd5c5f19287"} {"id":"scuba:admin_033_003","source_id":"scuba","original_id":"admin_033_003","kind":"benchmark_task","instruction":"Let’s enforce stronger credentials—set the minimum password length to 12 characters and require a mix of letters, numbers, and special symbols.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_003","query_templated":"Update the org's password policies as follows: set the minimum password length to 12 characters and must include alpha, numeric, special characters.","query":"Let’s enforce stronger credentials—set the minimum password length to 12 characters and require a mix of letters, numbers, and special symbols.","human_review_comment":"accept","ground_truth_dict":{"minimumPasswordLength":"12","complexity":"SpecialCharacters"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"set the minimum password length to 12 characters and must include alpha, numeric, special characters"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/143","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Password Policies","source_difficulty":"easy","template_id":"admin_033","template_name":"update_password_policies","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/143"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/143"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/update_password_policies"}],"source_schema_notes":[],"template_registry":{"name":"update_password_policies","original":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nLet’s enforce stronger credentials—set the minimum password length to 12 characters and require a mix of letters, numbers, and special symbols.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_003","query_templated":"Update the org's password policies as follows: set the minimum password length to 12 characters and must include alpha, numeric, special characters.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nLet’s enforce stronger credentials—set the minimum password length to 12 characters and require a mix of letters, numbers, and special symbols.\n```","human_review_comment":"accept","ground_truth_dict":{"minimumPasswordLength":"12","complexity":"SpecialCharacters"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"set the minimum password length to 12 characters and must include alpha, numeric, special characters"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/143"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L139","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_password_policies","data_retrieval_spec":[{"type":"metadata","id":"password_policy_info","params":{"types_and_members":{"Settings":["Security"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5395869bd01e3ad6e1ad2d4626be3787eb2b462652472e73699a60c618c9575e"} {"id":"scuba:admin_033_004","source_id":"scuba","original_id":"admin_033_004","kind":"benchmark_task","instruction":"Policy change request: set minimum password length to 15, force password change every 90 days, and history should keep the last 5 passwords.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_004","query_templated":"Update the org's password policies as follows: Minimum password length should be 15 characters, password should expire in 90 days, password history should remember last 5 passwords.","query":"Policy change request: set minimum password length to 15, force password change every 90 days, and history should keep the last 5 passwords.","human_review_comment":"rewrite","ground_truth_dict":{"minimumPasswordLength":"15","expiration":"ThreeMonths","historyRestriction":"5"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"Minimum password length should be 15 characters, password should expire in 90 days, password history should remember last 5 passwords"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/144","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Password Policies","source_difficulty":"easy","template_id":"admin_033","template_name":"update_password_policies","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/144"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/144"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/update_password_policies"}],"source_schema_notes":[],"template_registry":{"name":"update_password_policies","original":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPolicy change request: set minimum password length to 15, force password change every 90 days, and history should keep the last 5 passwords.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_004","query_templated":"Update the org's password policies as follows: Minimum password length should be 15 characters, password should expire in 90 days, password history should remember last 5 passwords.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPolicy change request: set minimum password length to 15, force password change every 90 days, and history should keep the last 5 passwords.\n```","human_review_comment":"rewrite","ground_truth_dict":{"minimumPasswordLength":"15","expiration":"ThreeMonths","historyRestriction":"5"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"Minimum password length should be 15 characters, password should expire in 90 days, password history should remember last 5 passwords"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/144"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L139","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_password_policies","data_retrieval_spec":[{"type":"metadata","id":"password_policy_info","params":{"types_and_members":{"Settings":["Security"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0dee24aabb8460c9453457af5112e72c82a79fd8ca56871e4388156b4f27b238"} {"id":"scuba:admin_033_005","source_id":"scuba","original_id":"admin_033_005","kind":"benchmark_task","instruction":"Update the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_005","query_templated":"Update the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.","query":"Update the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.","human_review_comment":"rewrite to use the origin","ground_truth_dict":{"minimumPasswordLength":"10","maxLoginAttempts":"FiveAttempts","lockoutInterval":"ThirtyMinutes"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/145","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Password Policies","source_difficulty":"easy","template_id":"admin_033","template_name":"update_password_policies","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/145"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/145"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/update_password_policies"}],"source_schema_notes":[],"template_registry":{"name":"update_password_policies","original":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/update_password_policies","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nUpdate the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Password Policies","task_id":"admin_033_005","query_templated":"Update the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.","query":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nUpdate the org's password policies as follows: Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes.\n```","human_review_comment":"rewrite to use the origin","ground_truth_dict":{"minimumPasswordLength":"10","maxLoginAttempts":"FiveAttempts","lockoutInterval":"ThirtyMinutes"},"difficulty":"easy","query_template_name":"update_password_policies","query_template_metadata":{"template_id":"admin_033","template_string":"Update the org's password policies as follows: {comma_separated_policy_conditions}.","prerequisites":{},"objects":[],"metadata_types":["Settings"]},"instance_dict":{"comma_separated_policy_conditions":"Minimum password length should be 10 characters, maximum invalid login attempts should be 5, lockout effective period should be 30 minutes"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Update the org's password polices as follows: update the [password expire days] to [180 days] and [set lockout effective period to forever].\nEXPERIENCE:\nSuccessful plan to update password policies:\n\n1. Log in to the org.\n2. Open Setup.\n3. In the Quick Find search box, type “password policies”.\n4. Select “Password Policies” from the results.\n5. Set “User passwords expire in” to “180 days”.\n6. Set “Lockout effective period” to “Forever”.\n7. Click “Save” to apply changes.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\"Update the org's password polices as follows: update the [Password complexity requirements] to [Must include numbers and uppercase and lowercase letters], and [Maximum invalid login attempts] to [5].\"\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the org.\n- Open Setup.\n- Use Quick Find to search for “Password Policies” and select it.\n- In Password Policies:\n - Set Password complexity requirements to “Must include numbers and uppercase and lowercase letters.”\n - Set Maximum invalid login attempts to 5.\n- Save the changes.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/145"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L139","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_password_policies","data_retrieval_spec":[{"type":"metadata","id":"password_policy_info","params":{"types_and_members":{"Settings":["Security"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"eb707f10645b558734a37a98ed8922500b1992f34bbbd301dc492471057f4eb8"} {"id":"scuba:admin_034_001","source_id":"scuba","original_id":"admin_034_001","kind":"benchmark_task","instruction":"Create a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_001","query_templated":"Create a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.","query":"Create a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.","human_review_comment":"reject; use original","ground_truth_dict":{"group_name":"Reviewer","users":["Security User"],"roles":["CEO"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Reviewer","comma_separated_users_and_roles":"\"Security\" User and \"CEO\" role"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/146","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Public Group","source_difficulty":"medium","template_id":"admin_034","template_name":"create_public_group","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/146"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/146"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_public_group"}],"source_schema_notes":[],"template_registry":{"name":"create_public_group","original":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nCreate a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_001","query_templated":"Create a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nCreate a Public group with the name Reviewer. Configure the public group members by adding \"Security\" User and \"CEO\" role.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"group_name":"Reviewer","users":["Security User"],"roles":["CEO"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Reviewer","comma_separated_users_and_roles":"\"Security\" User and \"CEO\" role"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/146"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L151","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_public_group","data_retrieval_spec":[{"type":"soql","id":"users","params":{"query":"SELECT Id, Name FROM User WHERE Name IN ({', '.join([\"'\"+username+\"'\" for username in users])})"}},{"type":"soql","id":"roles","params":{"query":"SELECT Id, Related.Name FROM Group WHERE Type='Role' AND Related.Name IN ({', '.join([\"'\"+rolename+\"'\" for rolename in roles])})"}},{"type":"soql","id":"add_members_to_group","params":{"query":"SELECT UserOrGroupId FROM GroupMember WHERE Group.Name='{group_name}'"}},{"type":"soql","id":"groups","params":{"query":"SELECT Id FROM Group WHERE Name='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2191615e897117e8210765f981d12b1a86b77c3b7b552bfd527ddcf38deb25ed"} {"id":"scuba:admin_034_002","source_id":"scuba","original_id":"admin_034_002","kind":"benchmark_task","instruction":"Please set up a Public Group called \"Finance Team\" and populate it with: - \"Auditor\" (User) - \"Accountant\" (User) - \"Controller\" (User) - \"CFO\" (Role) - \"Finance Manager\" (Role)","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_002","query_templated":"Create a Public group with the name Finance Team. Configure the public group members by adding \"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.","query":"Please set up a Public Group called \"Finance Team\" and populate it with: - \"Auditor\" (User) - \"Accountant\" (User) - \"Controller\" (User) - \"CFO\" (Role) - \"Finance Manager\" (Role)","human_review_comment":"accept","ground_truth_dict":{"group_name":"Finance Team","users":["Auditor","Accountant","Controller"],"roles":["CFO","Finance Manager"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Finance Team","comma_separated_users_and_roles":"\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/147","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Public Group","source_difficulty":"medium","template_id":"admin_034","template_name":"create_public_group","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/147"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/147"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_public_group"}],"source_schema_notes":[],"template_registry":{"name":"create_public_group","original":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nPlease set up a Public Group called \"Finance Team\" and populate it with: - \"Auditor\" (User) - \"Accountant\" (User) - \"Controller\" (User) - \"CFO\" (Role) - \"Finance Manager\" (Role)\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_002","query_templated":"Create a Public group with the name Finance Team. Configure the public group members by adding \"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nPlease set up a Public Group called \"Finance Team\" and populate it with: - \"Auditor\" (User) - \"Accountant\" (User) - \"Controller\" (User) - \"CFO\" (Role) - \"Finance Manager\" (Role)\n```","human_review_comment":"accept","ground_truth_dict":{"group_name":"Finance Team","users":["Auditor","Accountant","Controller"],"roles":["CFO","Finance Manager"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Finance Team","comma_separated_users_and_roles":"\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/147"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L151","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_public_group","data_retrieval_spec":[{"type":"soql","id":"users","params":{"query":"SELECT Id, Name FROM User WHERE Name IN ({', '.join([\"'\"+username+\"'\" for username in users])})"}},{"type":"soql","id":"roles","params":{"query":"SELECT Id, Related.Name FROM Group WHERE Type='Role' AND Related.Name IN ({', '.join([\"'\"+rolename+\"'\" for rolename in roles])})"}},{"type":"soql","id":"add_members_to_group","params":{"query":"SELECT UserOrGroupId FROM GroupMember WHERE Group.Name='{group_name}'"}},{"type":"soql","id":"groups","params":{"query":"SELECT Id FROM Group WHERE Name='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d378ba7d2b4e281c9e848f15536b477c39fd877affbd69f370c0db6a4ebe398c"} {"id":"scuba:admin_034_003","source_id":"scuba","original_id":"admin_034_003","kind":"benchmark_task","instruction":"Create a Public Group titled \"ProjectReviewers\" and make its members the \"Project Lead\" role and the \"Director, Operations\" role.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_003","query_templated":"Create a Public group with the name ProjectReviewers. Configure the public group members by adding \"Project Lead\" role and \"Director, Operations\" role.","query":"Create a Public Group titled \"ProjectReviewers\" and make its members the \"Project Lead\" role and the \"Director, Operations\" role.","human_review_comment":"accept","ground_truth_dict":{"group_name":"ProjectReviewers","users":[],"roles":["Project Lead","Director, Operations"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"ProjectReviewers","comma_separated_users_and_roles":"\"Project Lead\" role and \"Director, Operations\" role"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/148","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Public Group","source_difficulty":"medium","template_id":"admin_034","template_name":"create_public_group","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/148"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/148"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_public_group"}],"source_schema_notes":[],"template_registry":{"name":"create_public_group","original":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nCreate a Public Group titled \"ProjectReviewers\" and make its members the \"Project Lead\" role and the \"Director, Operations\" role.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_003","query_templated":"Create a Public group with the name ProjectReviewers. Configure the public group members by adding \"Project Lead\" role and \"Director, Operations\" role.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nCreate a Public Group titled \"ProjectReviewers\" and make its members the \"Project Lead\" role and the \"Director, Operations\" role.\n```","human_review_comment":"accept","ground_truth_dict":{"group_name":"ProjectReviewers","users":[],"roles":["Project Lead","Director, Operations"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"ProjectReviewers","comma_separated_users_and_roles":"\"Project Lead\" role and \"Director, Operations\" role"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/148"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L151","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_public_group","data_retrieval_spec":[{"type":"soql","id":"users","params":{"query":"SELECT Id, Name FROM User WHERE Name IN ({', '.join([\"'\"+username+\"'\" for username in users])})"}},{"type":"soql","id":"roles","params":{"query":"SELECT Id, Related.Name FROM Group WHERE Type='Role' AND Related.Name IN ({', '.join([\"'\"+rolename+\"'\" for rolename in roles])})"}},{"type":"soql","id":"add_members_to_group","params":{"query":"SELECT UserOrGroupId FROM GroupMember WHERE Group.Name='{group_name}'"}},{"type":"soql","id":"groups","params":{"query":"SELECT Id FROM Group WHERE Name='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f7ca4e6a791702449a0f1e4f6cba29d7f7e273943f691447ea02a85b6abba16c"} {"id":"scuba:admin_034_004","source_id":"scuba","original_id":"admin_034_004","kind":"benchmark_task","instruction":"Do me a favor and create a Public Group named \"Sales Team\"; include Alice Bob (user), the \"SVP, HR\" role, and \"Channel Sales Team\" as members.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_004","query_templated":"Create a Public group with the name Sales Team. Configure the public group members by adding Alice Bob user, SVP, HR role, Channel Sales Team.","query":"Do me a favor and create a Public Group named \"Sales Team\"; include Alice Bob (user), the \"SVP, HR\" role, and \"Channel Sales Team\" as members.","human_review_comment":"accept","ground_truth_dict":{"group_name":"Sales Team","users":["Alice Bob"],"roles":["SVP, Human Resources","Channel Sales Team"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Sales Team","comma_separated_users_and_roles":"Alice Bob user, SVP, HR role, Channel Sales Team"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/149","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Public Group","source_difficulty":"medium","template_id":"admin_034","template_name":"create_public_group","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/149"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/149"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_public_group"}],"source_schema_notes":[],"template_registry":{"name":"create_public_group","original":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nDo me a favor and create a Public Group named \"Sales Team\"; include Alice Bob (user), the \"SVP, HR\" role, and \"Channel Sales Team\" as members.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_004","query_templated":"Create a Public group with the name Sales Team. Configure the public group members by adding Alice Bob user, SVP, HR role, Channel Sales Team.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nDo me a favor and create a Public Group named \"Sales Team\"; include Alice Bob (user), the \"SVP, HR\" role, and \"Channel Sales Team\" as members.\n```","human_review_comment":"accept","ground_truth_dict":{"group_name":"Sales Team","users":["Alice Bob"],"roles":["SVP, Human Resources","Channel Sales Team"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Sales Team","comma_separated_users_and_roles":"Alice Bob user, SVP, HR role, Channel Sales Team"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/149"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L151","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_public_group","data_retrieval_spec":[{"type":"soql","id":"users","params":{"query":"SELECT Id, Name FROM User WHERE Name IN ({', '.join([\"'\"+username+\"'\" for username in users])})"}},{"type":"soql","id":"roles","params":{"query":"SELECT Id, Related.Name FROM Group WHERE Type='Role' AND Related.Name IN ({', '.join([\"'\"+rolename+\"'\" for rolename in roles])})"}},{"type":"soql","id":"add_members_to_group","params":{"query":"SELECT UserOrGroupId FROM GroupMember WHERE Group.Name='{group_name}'"}},{"type":"soql","id":"groups","params":{"query":"SELECT Id FROM Group WHERE Name='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"59bfd0a6419e92ffdeb36da94bb5070f2e532badef09c8a53ba31fb2639a180f"} {"id":"scuba:admin_034_005","source_id":"scuba","original_id":"admin_034_005","kind":"benchmark_task","instruction":"New Public Group creation request: call it \"Marketing Team\" and add the users \"Foo Bar\" and \"Sample User,\" plus the \"VP International Sales\" role.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_005","query_templated":"Create a Public group with the name Marketing Team. Configure the public group members by adding Foo Bar user, VP International Sales role, Sample User.","query":"New Public Group creation request: call it \"Marketing Team\" and add the users \"Foo Bar\" and \"Sample User,\" plus the \"VP International Sales\" role.","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"group_name":"Marketing Team","users":["Foo Bar"],"roles":["VP, International Sales"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Marketing Team","comma_separated_users_and_roles":"Foo Bar user, VP International Sales role, Sample User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/150","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Public Group","source_difficulty":"medium","template_id":"admin_034","template_name":"create_public_group","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/150"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/150"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_public_group"}],"source_schema_notes":[],"template_registry":{"name":"create_public_group","original":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_public_group","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nNew Public Group creation request: call it \"Marketing Team\" and add the users \"Foo Bar\" and \"Sample User,\" plus the \"VP International Sales\" role.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Public Group","task_id":"admin_034_005","query_templated":"Create a Public group with the name Marketing Team. Configure the public group members by adding Foo Bar user, VP International Sales role, Sample User.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory.\n\n## Your ultimate task\n```\nNew Public Group creation request: call it \"Marketing Team\" and add the users \"Foo Bar\" and \"Sample User,\" plus the \"VP International Sales\" role.\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"group_name":"Marketing Team","users":["Foo Bar"],"roles":["VP, International Sales"]},"difficulty":"medium","query_template_name":"create_public_group","query_template_metadata":{"template_id":"admin_034","template_string":"Create a Public group with the name {group_name}. Configure the public group members by adding {comma_separated_users_and_roles}.","prerequisites":{"data":{"objects":["UserRole"]}},"objects":["Group"],"metadata_types":[]},"instance_dict":{"group_name":"Marketing Team","comma_separated_users_and_roles":"Foo Bar user, VP International Sales role, Sample User"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Public group with the name [Reviewer]. Configure the public group members by adding [\"Security\" User and \"CEO\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Reviewer” and add members:\n\n- Log in, open Setup.\n- In Quick Find, type “Public Groups” and open Public Groups.\n- Click New.\n- Enter Label: “Reviewer” (Group Name auto-populates or click Group Name field if needed).\n- In the Members section:\n - Set the “Search” dropdown to Users, select “Security User,” click Add.\n - Change the “Search” dropdown to Roles, select “CEO,” click Add.\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:Create a Public group with the name [Finance Team]. Configure the public group members by adding [\"Auditor\" User, \"Accountant\" User, \"Controller\" User, \"CFO\" role, and \"Finance Manager\" role.]\nEXPERIENCE:\nSuccessful plan to create the Public Group “Finance Team” and add specified members:\n\n1) Log in to the org.\n2) Navigate: Setup > Users > Public Groups.\n3) Click New to create a public group.\n4) Set Label to “Finance Team” (click the Group Name field so it populates accordingly).\n5) In the “Search” dropdown, select Users. Select “Auditor User,” “Accountant User,” and “Controller User,” then click Add to move them to Selected Members.\n6) Change the “Search” dropdown to Roles. Select “CFO” and “Finance Manager,” then click Add to move them to Selected Members.\n7) Click Save.\n\nNotes:\n- No hot-keys were used in the successful trajectory."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/150"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L151","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_public_group","data_retrieval_spec":[{"type":"soql","id":"users","params":{"query":"SELECT Id, Name FROM User WHERE Name IN ({', '.join([\"'\"+username+\"'\" for username in users])})"}},{"type":"soql","id":"roles","params":{"query":"SELECT Id, Related.Name FROM Group WHERE Type='Role' AND Related.Name IN ({', '.join([\"'\"+rolename+\"'\" for rolename in roles])})"}},{"type":"soql","id":"add_members_to_group","params":{"query":"SELECT UserOrGroupId FROM GroupMember WHERE Group.Name='{group_name}'"}},{"type":"soql","id":"groups","params":{"query":"SELECT Id FROM Group WHERE Name='{group_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4bf80806510c8b20773a2803a4a0af63a35af9eb146cf8af780453e476cbfa63"} {"id":"scuba:admin_035_001","source_id":"scuba","original_id":"admin_035_001","kind":"benchmark_task","instruction":"Add a new CTO role to be a peer of the CEO (same level on the role hierarchy chart). Under the CTO, add a role titled \"VP in R&D\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_001","query_templated":"Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D..","query":"Add a new CTO role to be a peer of the CEO (same level on the role hierarchy chart). Under the CTO, add a role titled \"VP in R&D\".","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"CEO","parent_role_name":null,"is_new":false},{"role_name":"CTO","parent_role_name":null,"is_new":true},{"role_name":"VP in R&D","parent_role_name":"CTO","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/151","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Role Hierarchy","source_difficulty":"easy","template_id":"admin_035","template_name":"define_role_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/151"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/151"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/define_role_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"define_role_hierarchy","original":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nAdd a new CTO role to be a peer of the CEO (same level on the role hierarchy chart). Under the CTO, add a role titled \"VP in R&D\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_001","query_templated":"Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D..","query":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nAdd a new CTO role to be a peer of the CEO (same level on the role hierarchy chart). Under the CTO, add a role titled \"VP in R&D\".\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"CEO","parent_role_name":null,"is_new":false},{"role_name":"CTO","parent_role_name":null,"is_new":true},{"role_name":"VP in R&D","parent_role_name":"CTO","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/151"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L181","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_define_role_hierarchy","data_retrieval_spec":[{"type":"soql","id":"roles","params":{"query":"SELECT Name, ParentRole.Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"488ad7ccc2bcd7fd1e92ce4efaba8fe359ea425320c81d334f5079a75f3a6855"} {"id":"scuba:admin_035_002","source_id":"scuba","original_id":"admin_035_002","kind":"benchmark_task","instruction":"Please create \"Customer Support, New York\" role nested under the \"Customer Support, North America\" role. Then, beneath \"Customer Support, New York\" role, create a child role called \"Customer Service Agent\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_002","query_templated":"Create a Customer Support, New York role which should be under the Customer Support, North America role. Under the Customer Support, New York role, create a new role called Customer Service Agent..","query":"Please create \"Customer Support, New York\" role nested under the \"Customer Support, North America\" role. Then, beneath \"Customer Support, New York\" role, create a child role called \"Customer Service Agent\".","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"Customer Support, North America","parent_role_name":null,"is_new":true},{"role_name":"Customer Support, New York","parent_role_name":"Customer Support, North America","is_new":true},{"role_name":"Customer Service Agent","parent_role_name":"Customer Support, New York","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a Customer Support, New York role which should be under the Customer Support, North America role. Under the Customer Support, New York role, create a new role called Customer Service Agent."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/152","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Role Hierarchy","source_difficulty":"easy","template_id":"admin_035","template_name":"define_role_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/152"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/152"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/define_role_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"define_role_hierarchy","original":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nPlease create \"Customer Support, New York\" role nested under the \"Customer Support, North America\" role. Then, beneath \"Customer Support, New York\" role, create a child role called \"Customer Service Agent\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_002","query_templated":"Create a Customer Support, New York role which should be under the Customer Support, North America role. Under the Customer Support, New York role, create a new role called Customer Service Agent..","query":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nPlease create \"Customer Support, New York\" role nested under the \"Customer Support, North America\" role. Then, beneath \"Customer Support, New York\" role, create a child role called \"Customer Service Agent\".\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"Customer Support, North America","parent_role_name":null,"is_new":true},{"role_name":"Customer Support, New York","parent_role_name":"Customer Support, North America","is_new":true},{"role_name":"Customer Service Agent","parent_role_name":"Customer Support, New York","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a Customer Support, New York role which should be under the Customer Support, North America role. Under the Customer Support, New York role, create a new role called Customer Service Agent."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/152"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L181","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_define_role_hierarchy","data_retrieval_spec":[{"type":"soql","id":"roles","params":{"query":"SELECT Name, ParentRole.Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c87b5be9a9c669c76698a1d96977fb5cdb85697030dbbb26a1756355db2b51ce"} {"id":"scuba:admin_035_003","source_id":"scuba","original_id":"admin_035_003","kind":"benchmark_task","instruction":"Org chart update: create a \"Chief Procurement Officer\" that reports to the CFO, and under that CPO role add \"Purchasing Manager\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_003","query_templated":"Create a Chief Procurement Officer role which shoule be under the CFO. Under the Chief Procurement Officer, create a new role called Purchasing Manager..","query":"Org chart update: create a \"Chief Procurement Officer\" that reports to the CFO, and under that CPO role add \"Purchasing Manager\".","human_review_comment":"accept","ground_truth_dict":{"hierarchy":[{"role_name":"Chief Procurement Officer","parent_role_name":"CFO","is_new":true},{"role_name":"Purchasing Manager","parent_role_name":"Chief Procurement Officer","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a Chief Procurement Officer role which shoule be under the CFO. Under the Chief Procurement Officer, create a new role called Purchasing Manager."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/153","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Role Hierarchy","source_difficulty":"easy","template_id":"admin_035","template_name":"define_role_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/153"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/153"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/define_role_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"define_role_hierarchy","original":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nOrg chart update: create a \"Chief Procurement Officer\" that reports to the CFO, and under that CPO role add \"Purchasing Manager\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_003","query_templated":"Create a Chief Procurement Officer role which shoule be under the CFO. Under the Chief Procurement Officer, create a new role called Purchasing Manager..","query":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nOrg chart update: create a \"Chief Procurement Officer\" that reports to the CFO, and under that CPO role add \"Purchasing Manager\".\n```","human_review_comment":"accept","ground_truth_dict":{"hierarchy":[{"role_name":"Chief Procurement Officer","parent_role_name":"CFO","is_new":true},{"role_name":"Purchasing Manager","parent_role_name":"Chief Procurement Officer","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a Chief Procurement Officer role which shoule be under the CFO. Under the Chief Procurement Officer, create a new role called Purchasing Manager."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/153"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L181","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_define_role_hierarchy","data_retrieval_spec":[{"type":"soql","id":"roles","params":{"query":"SELECT Name, ParentRole.Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"059c9f247b163654e92551811db033c474201b3c2bd3a139a71bc3c17db532a5"} {"id":"scuba:admin_035_004","source_id":"scuba","original_id":"admin_035_004","kind":"benchmark_task","instruction":"Set the role hierarchy chart so the Sales Manager reports to the Vice President of Sales, and the Vice President of Sales reports to the CEO.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_004","query_templated":"Change the current role hierarchy such that: CEO > Vice President of Sales > Sales Manager.","query":"Set the role hierarchy chart so the Sales Manager reports to the Vice President of Sales, and the Vice President of Sales reports to the CEO.","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"CEO","parent_role_name":null,"is_new":false},{"role_name":"Vice President of Sales","parent_role_name":"CEO","is_new":false},{"role_name":"Sales Manager","parent_role_name":"Vice President of Sales","is_new":false}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Change the current role hierarchy such that: CEO > Vice President of Sales > Sales Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/154","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Role Hierarchy","source_difficulty":"easy","template_id":"admin_035","template_name":"define_role_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/154"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/154"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/define_role_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"define_role_hierarchy","original":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nSet the role hierarchy chart so the Sales Manager reports to the Vice President of Sales, and the Vice President of Sales reports to the CEO.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_004","query_templated":"Change the current role hierarchy such that: CEO > Vice President of Sales > Sales Manager.","query":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nSet the role hierarchy chart so the Sales Manager reports to the Vice President of Sales, and the Vice President of Sales reports to the CEO.\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"hierarchy":[{"role_name":"CEO","parent_role_name":null,"is_new":false},{"role_name":"Vice President of Sales","parent_role_name":"CEO","is_new":false},{"role_name":"Sales Manager","parent_role_name":"Vice President of Sales","is_new":false}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Change the current role hierarchy such that: CEO > Vice President of Sales > Sales Manager"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/154"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L181","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_define_role_hierarchy","data_retrieval_spec":[{"type":"soql","id":"roles","params":{"query":"SELECT Name, ParentRole.Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"79f46ba8f2fb3acb7ffaf3b3a73bec6dec7da92c0852637a7c225ede5c231877"} {"id":"scuba:admin_035_005","source_id":"scuba","original_id":"admin_035_005","kind":"benchmark_task","instruction":"Create a role named \"Associate Sales Representative\" and have it report to \"SVP, Sales & Marketing\". Also add two direct reports under it: \"Sales Intern\" and \"Sales Trainee\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_005","query_templated":"Create a new role called 'Associate Sales Representative' and make it report to 'SVP, Sales & Marketing'. Additionally, create roles called 'Sales Intern' and 'Sales Trainee', both reporting to 'Associate Sales Representative'..","query":"Create a role named \"Associate Sales Representative\" and have it report to \"SVP, Sales & Marketing\". Also add two direct reports under it: \"Sales Intern\" and \"Sales Trainee\".","human_review_comment":"good! accept","ground_truth_dict":{"hierarchy":[{"role_name":"Associate Sales Representative","parent_role_name":"SVP, Sales & Marketing","is_new":true},{"role_name":"Sales Intern","parent_role_name":"Associate Sales Representative","is_new":true},{"role_name":"Sales Trainee","parent_role_name":"Associate Sales Representative","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a new role called 'Associate Sales Representative' and make it report to 'SVP, Sales & Marketing'. Additionally, create roles called 'Sales Intern' and 'Sales Trainee', both reporting to 'Associate Sales Representative'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/155","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Role Hierarchy","source_difficulty":"easy","template_id":"admin_035","template_name":"define_role_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/155"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/155"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/define_role_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"define_role_hierarchy","original":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/define_role_hierarchy","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nCreate a role named \"Associate Sales Representative\" and have it report to \"SVP, Sales & Marketing\". Also add two direct reports under it: \"Sales Intern\" and \"Sales Trainee\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Role Hierarchy","task_id":"admin_035_005","query_templated":"Create a new role called 'Associate Sales Representative' and make it report to 'SVP, Sales & Marketing'. Additionally, create roles called 'Sales Intern' and 'Sales Trainee', both reporting to 'Associate Sales Representative'..","query":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first.\n\n## Your ultimate task\n```\nCreate a role named \"Associate Sales Representative\" and have it report to \"SVP, Sales & Marketing\". Also add two direct reports under it: \"Sales Intern\" and \"Sales Trainee\".\n```","human_review_comment":"good! accept","ground_truth_dict":{"hierarchy":[{"role_name":"Associate Sales Representative","parent_role_name":"SVP, Sales & Marketing","is_new":true},{"role_name":"Sales Intern","parent_role_name":"Associate Sales Representative","is_new":true},{"role_name":"Sales Trainee","parent_role_name":"Associate Sales Representative","is_new":true}]},"difficulty":"easy","query_template_name":"define_role_hierarchy","query_template_metadata":{"template_id":"admin_035","template_string":"{org_hierarchy_description}.","prerequisites":{},"objects":["UserRole"],"metadata_types":[]},"instance_dict":{"org_hierarchy_description":"Create a new role called 'Associate Sales Representative' and make it report to 'SVP, Sales & Marketing'. Additionally, create roles called 'Sales Intern' and 'Sales Trainee', both reporting to 'Associate Sales Representative'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:[Create a CTO role which shoule be in the same level as the CEO. Under the CTO, create a new role called VP in R&D].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, type “roles” and open Roles > Set Up Roles.\n- Create CTO as a top-level role (peer to CEO):\n - Click Add Role.\n - Enter Label: CTO (let Role Name auto-populate).\n - Save.\n- Create VP in R&D under CTO:\n - Return to Roles > Set Up Roles.\n - Select the CTO node and click Add Role (to create a child role under CTO).\n - Enter Label: VP in R&D (let Role Name auto-populate).\n - Save.\n\nNotes:\n- No hotkeys were used.\n- The key placement actions are: add CTO at the top level (same level as CEO), then add VP in R&D as a child of CTO.\nSample 2:\nTASK:Create a [Customer Support, New York] role which should be under the [Customer Support, North America] role. Under the [Customer Support, New York] role, create a new role called [Customer Service Agent].\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, search for “users,” then navigate to Users > Roles > Set Up Roles.\n- Expand the role hierarchy and use Add Role (from the appropriate parent context) to create a child role:\n - Enter Label: “Customer Support, New York” (Role Name auto-populates after focusing the Role Name field).\n - Click Save & New.\n- Create the next role under the newly created role:\n - Enter Label: “Customer Service Agent” (focus Role Name field to auto-populate).\n - Set parent via Lookup on “This role reports to” and select “Customer Support, New York.”\n - Click Save.\n\nNotes:\n- No hotkeys were used.\n- “Save & New” was used to chain creation of the second role immediately after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/155"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L181","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_define_role_hierarchy","data_retrieval_spec":[{"type":"soql","id":"roles","params":{"query":"SELECT Name, ParentRole.Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"88c056aa2bc95869a943e07777b7b1358450f423423b4137aa86f3e93ebbdfb5"} {"id":"scuba:admin_037_001","source_id":"scuba","original_id":"admin_037_001","kind":"benchmark_task","instruction":"Please set up an approval process on the Opportunity object using the Jump Start Wizard. Name it \"Approve Opportunity Amount\" and have it fire only when the Opportunity Amount is greater than 200.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_001","query_templated":"Build an approval process for the records in the Opportunity Object using the Jump Start Wizard. Name this process as Approve Opportunity Amount. The entry criteria is set to the opportunity amount is greater than 200.","query":"Please set up an approval process on the Opportunity object using the Jump Start Wizard. Name it \"Approve Opportunity Amount\" and have it fire only when the Opportunity Amount is greater than 200.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","approval_process_name":"Approve Opportunity Amount","entry_criteria_formula":[["Opportunity.Amount","greaterThan","200"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Opportunity","approval_process_name":"Approve Opportunity Amount","entry_criteria_formula":"the opportunity amount is greater than 200"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/156","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"easy","template_id":"admin_037","template_name":"create_approval_process_jump_start","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/156"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/156"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_approval_process_jump_start"}],"source_schema_notes":[],"template_registry":{"name":"create_approval_process_jump_start","original":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nPlease set up an approval process on the Opportunity object using the Jump Start Wizard. Name it \"Approve Opportunity Amount\" and have it fire only when the Opportunity Amount is greater than 200.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_001","query_templated":"Build an approval process for the records in the Opportunity Object using the Jump Start Wizard. Name this process as Approve Opportunity Amount. The entry criteria is set to the opportunity amount is greater than 200.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nPlease set up an approval process on the Opportunity object using the Jump Start Wizard. Name it \"Approve Opportunity Amount\" and have it fire only when the Opportunity Amount is greater than 200.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Opportunity","approval_process_name":"Approve Opportunity Amount","entry_criteria_formula":[["Opportunity.Amount","greaterThan","200"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Opportunity","approval_process_name":"Approve Opportunity Amount","entry_criteria_formula":"the opportunity amount is greater than 200"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/156"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L201","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_approval_process_jump_start","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a1b4349fe024881b7e630c7b99a6947e29de3d54fcdcd6ec537a93c0821ac0c6"} {"id":"scuba:admin_037_002","source_id":"scuba","original_id":"admin_037_002","kind":"benchmark_task","instruction":"Can you create an approval process for Account records via the Jump Start Wizard and call it \"Approve Low Earning Account\"? It should only enter approval if the Billing: Country is not \"United States\" and the Annual Revenue is less than 100,000.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_002","query_templated":"Build an approval process for the records in the Account Object using the Jump Start Wizard. Name this process as Approve Low Earning Account. The entry criteria is set to Account: Billing Country not equal to United States and Account: Annual Revenue is less than 100000.","query":"Can you create an approval process for Account records via the Jump Start Wizard and call it \"Approve Low Earning Account\"? It should only enter approval if the Billing: Country is not \"United States\" and the Annual Revenue is less than 100,000.","human_review_comment":"minor tweak","ground_truth_dict":{"object_name":"Account","approval_process_name":"Approve Low Earning Account","entry_criteria_formula":[["Account.BillingCountry","notEqual","United States"],["Account.AnnualRevenue","lessThan","100000"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Account","approval_process_name":"Approve Low Earning Account","entry_criteria_formula":"Account: Billing Country not equal to United States and Account: Annual Revenue is less than 100000"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/157","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"easy","template_id":"admin_037","template_name":"create_approval_process_jump_start","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/157"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/157"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_approval_process_jump_start"}],"source_schema_notes":[],"template_registry":{"name":"create_approval_process_jump_start","original":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nCan you create an approval process for Account records via the Jump Start Wizard and call it \"Approve Low Earning Account\"? It should only enter approval if the Billing: Country is not \"United States\" and the Annual Revenue is less than 100,000.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_002","query_templated":"Build an approval process for the records in the Account Object using the Jump Start Wizard. Name this process as Approve Low Earning Account. The entry criteria is set to Account: Billing Country not equal to United States and Account: Annual Revenue is less than 100000.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nCan you create an approval process for Account records via the Jump Start Wizard and call it \"Approve Low Earning Account\"? It should only enter approval if the Billing: Country is not \"United States\" and the Annual Revenue is less than 100,000.\n```","human_review_comment":"minor tweak","ground_truth_dict":{"object_name":"Account","approval_process_name":"Approve Low Earning Account","entry_criteria_formula":[["Account.BillingCountry","notEqual","United States"],["Account.AnnualRevenue","lessThan","100000"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Account","approval_process_name":"Approve Low Earning Account","entry_criteria_formula":"Account: Billing Country not equal to United States and Account: Annual Revenue is less than 100000"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/157"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L201","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_approval_process_jump_start","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0b7598c0a86879ff98905bb97f197dd524cc2cbe0f62efcbd3608d01cd43fa29"} {"id":"scuba:admin_037_003","source_id":"scuba","original_id":"admin_037_003","kind":"benchmark_task","instruction":"I’d like a Case approval built with the Jump Start Wizard named \"Approve Working Case.\" Configure the entry criteria so it starts when Case: Status equals \"Working,\" the Case: Description contains \"new,\" and Case: Business Hours starts with \"Default.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_003","query_templated":"Build an approval process for the records in the Case Object using the Jump Start Wizard. Name this process as Approve Working Case. The entry criteria is set to Case: Status equals Working, Case: Description contains new, Case: Business Hours starts with Default.","query":"I’d like a Case approval built with the Jump Start Wizard named \"Approve Working Case.\" Configure the entry criteria so it starts when Case: Status equals \"Working,\" the Case: Description contains \"new,\" and Case: Business Hours starts with \"Default.\"","human_review_comment":"modified multiple palces","ground_truth_dict":{"object_name":"Case","approval_process_name":"Approve Working Case","entry_criteria_formula":[["Case.Status","equals","Working"],["Case.Description","contains","new"],["Case.BusinessHours","startsWith","Default"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Case","approval_process_name":"Approve Working Case","entry_criteria_formula":"Case: Status equals Working, Case: Description contains new, Case: Business Hours starts with Default"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/158","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"easy","template_id":"admin_037","template_name":"create_approval_process_jump_start","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/158"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/158"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_approval_process_jump_start"}],"source_schema_notes":[],"template_registry":{"name":"create_approval_process_jump_start","original":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nI’d like a Case approval built with the Jump Start Wizard named \"Approve Working Case.\" Configure the entry criteria so it starts when Case: Status equals \"Working,\" the Case: Description contains \"new,\" and Case: Business Hours starts with \"Default.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_003","query_templated":"Build an approval process for the records in the Case Object using the Jump Start Wizard. Name this process as Approve Working Case. The entry criteria is set to Case: Status equals Working, Case: Description contains new, Case: Business Hours starts with Default.","query":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nI’d like a Case approval built with the Jump Start Wizard named \"Approve Working Case.\" Configure the entry criteria so it starts when Case: Status equals \"Working,\" the Case: Description contains \"new,\" and Case: Business Hours starts with \"Default.\"\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"object_name":"Case","approval_process_name":"Approve Working Case","entry_criteria_formula":[["Case.Status","equals","Working"],["Case.Description","contains","new"],["Case.BusinessHours","startsWith","Default"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Case","approval_process_name":"Approve Working Case","entry_criteria_formula":"Case: Status equals Working, Case: Description contains new, Case: Business Hours starts with Default"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/158"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L201","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_approval_process_jump_start","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e28d66d1cdc167ea47d93962308dc606cbe733d32048794753cfbc62cabe951e"} {"id":"scuba:admin_037_004","source_id":"scuba","original_id":"admin_037_004","kind":"benchmark_task","instruction":"For the Case object, use the Jump Start Wizard to make an approval process called \"case_approval.\" It should kick off when Status is \"Escalated\" and the Description field includes \"APT.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_004","query_templated":"Build an approval process for the records in the Case Object using the Jump Start Wizard. Name this process as case_approval. The entry criteria is set to The approval process should start when the Status is 'Escalated' and the Description has 'APT' in it..","query":"For the Case object, use the Jump Start Wizard to make an approval process called \"case_approval.\" It should kick off when Status is \"Escalated\" and the Description field includes \"APT.\"","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","approval_process_name":"case_approval","entry_criteria_formula":[["Case.Status","equals","Escalated"],["Case.Description","contains","APT"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Case","approval_process_name":"case_approval","entry_criteria_formula":"The approval process should start when the Status is 'Escalated' and the Description has 'APT' in it."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/159","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"easy","template_id":"admin_037","template_name":"create_approval_process_jump_start","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/159"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/159"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_approval_process_jump_start"}],"source_schema_notes":[],"template_registry":{"name":"create_approval_process_jump_start","original":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nFor the Case object, use the Jump Start Wizard to make an approval process called \"case_approval.\" It should kick off when Status is \"Escalated\" and the Description field includes \"APT.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_004","query_templated":"Build an approval process for the records in the Case Object using the Jump Start Wizard. Name this process as case_approval. The entry criteria is set to The approval process should start when the Status is 'Escalated' and the Description has 'APT' in it..","query":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nFor the Case object, use the Jump Start Wizard to make an approval process called \"case_approval.\" It should kick off when Status is \"Escalated\" and the Description field includes \"APT.\"\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Case","approval_process_name":"case_approval","entry_criteria_formula":[["Case.Status","equals","Escalated"],["Case.Description","contains","APT"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Case","approval_process_name":"case_approval","entry_criteria_formula":"The approval process should start when the Status is 'Escalated' and the Description has 'APT' in it."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/159"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L201","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_approval_process_jump_start","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"840e75ea1edb33b55e36d010d72c255ec763ac6dfa391aa12de5cc39ad7d91ce"} {"id":"scuba:admin_037_005","source_id":"scuba","original_id":"admin_037_005","kind":"benchmark_task","instruction":"On the Account object, build an approval process through the Jump Start Wizard named \"account_approval.\" Trigger it only when Type is \"Customer - Direct\" and Annual Revenue is greater than 500,000.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_005","query_templated":"Build an approval process for the records in the Account Object using the Jump Start Wizard. Name this process as account_approval. The entry criteria is set to The approval process should start when the Type is 'Customer - Direct' and the Annual Revenue is greater than 500000..","query":"On the Account object, build an approval process through the Jump Start Wizard named \"account_approval.\" Trigger it only when Type is \"Customer - Direct\" and Annual Revenue is greater than 500,000.","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","approval_process_name":"account_approval","entry_criteria_formula":[["Account.Type","equals","Customer - Direct"],["Account.AnnualRevenue","greaterThan","500000"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Account","approval_process_name":"account_approval","entry_criteria_formula":"The approval process should start when the Type is 'Customer - Direct' and the Annual Revenue is greater than 500000."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/160","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"easy","template_id":"admin_037","template_name":"create_approval_process_jump_start","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/160"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/160"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/create_approval_process_jump_start"}],"source_schema_notes":[],"template_registry":{"name":"create_approval_process_jump_start","original":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/create_approval_process_jump_start","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nOn the Account object, build an approval process through the Jump Start Wizard named \"account_approval.\" Trigger it only when Type is \"Customer - Direct\" and Annual Revenue is greater than 500,000.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_037_005","query_templated":"Build an approval process for the records in the Account Object using the Jump Start Wizard. Name this process as account_approval. The entry criteria is set to The approval process should start when the Type is 'Customer - Direct' and the Annual Revenue is greater than 500000..","query":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save.\n\n## Your ultimate task\n```\nOn the Account object, build an approval process through the Jump Start Wizard named \"account_approval.\" Trigger it only when Type is \"Customer - Direct\" and Annual Revenue is greater than 500,000.\n```","human_review_comment":"accept","ground_truth_dict":{"object_name":"Account","approval_process_name":"account_approval","entry_criteria_formula":[["Account.Type","equals","Customer - Direct"],["Account.AnnualRevenue","greaterThan","500000"]]},"difficulty":"easy","query_template_name":"create_approval_process_jump_start","query_template_metadata":{"template_id":"admin_037","template_string":"Build an approval process for the records in the {object_name} Object using the Jump Start Wizard. Name this process as {approval_process_name}. The entry criteria is set to {entry_criteria_formula}.","prerequisites":{},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"object_name":"Account","approval_process_name":"account_approval","entry_criteria_formula":"The approval process should start when the Type is 'Customer - Direct' and the Annual Revenue is greater than 500000."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Build an approval process for the records in the [Opportunity] Object using the Jump Start Wizard. Name this process as [Approve Opportunity Amount]. The entry criteria is set to [the opportunity amount is greater than 200].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in and open Setup.\n- In Quick Find, search for “Approval Processes” and open it.\n- Set “Manage Approval Processes For” to Opportunity.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Enter Process Name: “Approve Opportunity Amount.”\n- Click into the Unique Name field to auto-populate it.\n- Define entry criteria:\n - Field: Opportunity: Amount\n - Operator: greater than\n - Value: 200\n- Click Save.\nSample 2:\nTASK:Build an approval process for the records in the [Account] Object using the Jump Start Wizard. Name this process as [Approve Low Earning Account]. The entry criteria is set to [Account: Billing Country not equal to United States] and [Account: Annual Revenue is less than 100000].\nEXPERIENCE:\nSuccessful plan to build the approval process:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Approval Processes” and navigate to it.\n- In “Manage Approval Processes For,” select Account.\n- Click “Create New Approval Process” and choose “Use Jump Start Wizard.”\n- Set Process Name to “Approve Low Earning Account.” (Leave Unique Name auto-populated.)\n- Define entry criteria with two conditions:\n - Field: Account: Billing Country | Operator: not equal to | Value: United States\n - Field: Account: Annual Revenue | Operator: less than | Value: 100000\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/160"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L201","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_approval_process_jump_start","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"05d2c70872c0c636d2085d9bfcd89cc5f6083aa03c690c825514813756c1e498"} {"id":"scuba:admin_038_001","source_id":"scuba","original_id":"admin_038_001","kind":"benchmark_task","instruction":"In the required_account_action approval process, adding new actions like this: under Final Approval Actions, add a field update named mark_clean that sets the Clean Status field to In Sync. Under Final Rejection Actions, add another field update named mark_inactive that sets Clean Status to Inactive.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_001","query_templated":"Configure the required_account_action approval process's final approval actions by adding a new action with the name mark_clean, which set the Clean Status field value to In Sync. Similarly, add a new action under the final rejection actions with the name mark_inactive, which set the field value to Inactive.","query":"In the required_account_action approval process, adding new actions like this: under Final Approval Actions, add a field update named mark_clean that sets the Clean Status field to In Sync. Under Final Rejection Actions, add another field update named mark_inactive that sets Clean Status to Inactive.","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"approval_process_name":"Account.required_account_action","final_approval_action_name":"mark_clean","final_rejection_action_name":"mark_inactive","approval_status_field":"CleanStatus","approved_value":"Matched","rejected_value":"Inactive"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess","Workflow"]},"instance_dict":{"approval_process_name":"required_account_action","final_approval_action_name":"mark_clean","final_rejection_action_name":"mark_inactive","approval_status_field":"Clean Status","approved_value":"In Sync","rejected_value":"Inactive"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/161","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"hard","template_id":"admin_038","template_name":"configure_approval_process_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/161"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/161"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/configure_approval_process_actions"}],"source_schema_notes":[],"template_registry":{"name":"configure_approval_process_actions","original":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nIn the required_account_action approval process, adding new actions like this: under Final Approval Actions, add a field update named mark_clean that sets the Clean Status field to In Sync. Under Final Rejection Actions, add another field update named mark_inactive that sets Clean Status to Inactive.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_001","query_templated":"Configure the required_account_action approval process's final approval actions by adding a new action with the name mark_clean, which set the Clean Status field value to In Sync. Similarly, add a new action under the final rejection actions with the name mark_inactive, which set the field value to Inactive.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nIn the required_account_action approval process, adding new actions like this: under Final Approval Actions, add a field update named mark_clean that sets the Clean Status field to In Sync. Under Final Rejection Actions, add another field update named mark_inactive that sets Clean Status to Inactive.\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"approval_process_name":"Account.required_account_action","final_approval_action_name":"mark_clean","final_rejection_action_name":"mark_inactive","approval_status_field":"CleanStatus","approved_value":"Matched","rejected_value":"Inactive"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess","Workflow"]},"instance_dict":{"approval_process_name":"required_account_action","final_approval_action_name":"mark_clean","final_rejection_action_name":"mark_inactive","approval_status_field":"Clean Status","approved_value":"In Sync","rejected_value":"Inactive"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/161"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L240","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_configure_approval_process_actions","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}},{"type":"metadata","id":"workflows","params":{"types_and_members":{"Workflow":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"667c03f019a74d643dc647a01c1ab04d5745439130a91e0b0a20655577873214"} {"id":"scuba:admin_038_002","source_id":"scuba","original_id":"admin_038_002","kind":"benchmark_task","instruction":"Configure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_002","query_templated":"Configure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.","query":"Configure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"approval_process_name":"Product2.required_product_action","final_approval_action_name":"activate","final_rejection_action_name":"deserialize","approval_status_field":"Active","approved_value":"True","rejected_value":"False"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"required_product_action","final_approval_action_name":"activate","final_rejection_action_name":"deserialize","approval_status_field":"Active","approved_value":true,"rejected_value":false},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/162","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"hard","template_id":"admin_038","template_name":"configure_approval_process_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/162"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/162"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/configure_approval_process_actions"}],"source_schema_notes":[],"template_registry":{"name":"configure_approval_process_actions","original":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nConfigure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_002","query_templated":"Configure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nConfigure the required_product_action approval process's final approval actions by adding a new action with the name activate, which set the Active field value to True. Similarly, add a new action under the final rejection actions with the name deserialize, which set the field value to False.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"approval_process_name":"Product2.required_product_action","final_approval_action_name":"activate","final_rejection_action_name":"deserialize","approval_status_field":"Active","approved_value":"True","rejected_value":"False"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"required_product_action","final_approval_action_name":"activate","final_rejection_action_name":"deserialize","approval_status_field":"Active","approved_value":true,"rejected_value":false},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/162"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L240","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_configure_approval_process_actions","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}},{"type":"metadata","id":"workflows","params":{"types_and_members":{"Workflow":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5b2c2bdd5369d1204f114dc2184fb0550b6970b525fff3b0162683a951fc43ba"} {"id":"scuba:admin_038_003","source_id":"scuba","original_id":"admin_038_003","kind":"benchmark_task","instruction":"In Campaign.required_campaign_action approval process, do the following: 1) Under Final Approval Actions — add a new action named Approve Campaign to update the Status field to Planned. 2) Under Final Rejection Actions — add a action named Reject Campaign to update Status to Aborted.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_003","query_templated":"Configure the Campaign.required_campaign_action approval process's final approval actions by adding a new action with the name Approve Campaign, which set the Status field value to Planned. Similarly, add a new action under the final rejection actions with the name Reject Campaign, which set the field value to Aborted.","query":"In Campaign.required_campaign_action approval process, do the following: 1) Under Final Approval Actions — add a new action named Approve Campaign to update the Status field to Planned. 2) Under Final Rejection Actions — add a action named Reject Campaign to update Status to Aborted.","human_review_comment":"modified multiple palces","ground_truth_dict":{"approval_process_name":"Campaign.required_campaign_action","final_approval_action_name":"Approve Campaign","final_rejection_action_name":"Reject Campaign","approval_status_field":"Status","approved_value":"Planned","rejected_value":"Aborted"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Campaign.required_campaign_action","final_approval_action_name":"Approve Campaign","final_rejection_action_name":"Reject Campaign","approval_status_field":"Status","approved_value":"Planned","rejected_value":"Aborted"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/163","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"hard","template_id":"admin_038","template_name":"configure_approval_process_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/163"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/163"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/configure_approval_process_actions"}],"source_schema_notes":[],"template_registry":{"name":"configure_approval_process_actions","original":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nIn Campaign.required_campaign_action approval process, do the following: 1) Under Final Approval Actions — add a new action named Approve Campaign to update the Status field to Planned. 2) Under Final Rejection Actions — add a action named Reject Campaign to update Status to Aborted.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_003","query_templated":"Configure the Campaign.required_campaign_action approval process's final approval actions by adding a new action with the name Approve Campaign, which set the Status field value to Planned. Similarly, add a new action under the final rejection actions with the name Reject Campaign, which set the field value to Aborted.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nIn Campaign.required_campaign_action approval process, do the following: 1) Under Final Approval Actions — add a new action named Approve Campaign to update the Status field to Planned. 2) Under Final Rejection Actions — add a action named Reject Campaign to update Status to Aborted.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"approval_process_name":"Campaign.required_campaign_action","final_approval_action_name":"Approve Campaign","final_rejection_action_name":"Reject Campaign","approval_status_field":"Status","approved_value":"Planned","rejected_value":"Aborted"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Campaign.required_campaign_action","final_approval_action_name":"Approve Campaign","final_rejection_action_name":"Reject Campaign","approval_status_field":"Status","approved_value":"Planned","rejected_value":"Aborted"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/163"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L240","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_configure_approval_process_actions","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}},{"type":"metadata","id":"workflows","params":{"types_and_members":{"Workflow":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"ff0801cdad5ed59bf1c7aed05221ee05ac0a096938fa44efa7aa7cb7b4a7d7c1"} {"id":"scuba:admin_038_004","source_id":"scuba","original_id":"admin_038_004","kind":"benchmark_task","instruction":"Configure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_004","query_templated":"Configure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".","query":"Configure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".","human_review_comment":"reject; use the orginal","ground_truth_dict":{"approval_process_name":"Contact.required_contact_action","final_approval_action_name":"Add to Description","final_rejection_action_name":"Empty Description","approval_status_field":"Description","approved_value":"Description + \"This is a valid contact\"","rejected_value":"\"Rejected Contact\""},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Contact.required_contact_action","final_approval_action_name":"Add to Description","final_rejection_action_name":"Empty Description","approval_status_field":"Description","approved_value":"Formula of Description + \"This is a valid contact\"","rejected_value":"Rejected Contact"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/164","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"hard","template_id":"admin_038","template_name":"configure_approval_process_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/164"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/164"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/configure_approval_process_actions"}],"source_schema_notes":[],"template_registry":{"name":"configure_approval_process_actions","original":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nConfigure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_004","query_templated":"Configure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nConfigure the Contact.required_contact_action approval process's final approval actions by adding a new action with the name Add to Description, which set the Description field value to Formula of Description + \"This is a valid contact\". Similarly, add a new action under the final rejection actions with the name Empty Description, which set the field value to \"Rejected Contact\".\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"approval_process_name":"Contact.required_contact_action","final_approval_action_name":"Add to Description","final_rejection_action_name":"Empty Description","approval_status_field":"Description","approved_value":"Description + \"This is a valid contact\"","rejected_value":"\"Rejected Contact\""},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Contact.required_contact_action","final_approval_action_name":"Add to Description","final_rejection_action_name":"Empty Description","approval_status_field":"Description","approved_value":"Formula of Description + \"This is a valid contact\"","rejected_value":"Rejected Contact"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/164"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L240","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_configure_approval_process_actions","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}},{"type":"metadata","id":"workflows","params":{"types_and_members":{"Workflow":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"65e8da615f626d6a4c3744dc38431d51f128604f25c8ed79541678eaeeb59eda"} {"id":"scuba:admin_038_005","source_id":"scuba","original_id":"admin_038_005","kind":"benchmark_task","instruction":"For the Lead.required_lead_action approval process, make the end-of-process steps: on final approval, build a new action named Update Status to set the Status field to Working - Contacted; on final rejection, configure a new action named Close Lead to set Status to Closed - Not Converted.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_005","query_templated":"Configure the Lead.required_lead_action approval process's final approval actions by adding a new action with the name Update Status, which set the Status field value to Working - Contacted. Similarly, add a new action under the final rejection actions with the name Close Lead, which set the field value to Closed - Not Converted.","query":"For the Lead.required_lead_action approval process, make the end-of-process steps: on final approval, build a new action named Update Status to set the Status field to Working - Contacted; on final rejection, configure a new action named Close Lead to set Status to Closed - Not Converted.","human_review_comment":"modified multiple palces","ground_truth_dict":{"approval_process_name":"Lead.required_lead_action","final_approval_action_name":"Update Status","final_rejection_action_name":"Close Lead","approval_status_field":"Status","approved_value":"Working - Contacted","rejected_value":"Closed - Not Converted"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Lead.required_lead_action","final_approval_action_name":"Update Status","final_rejection_action_name":"Close Lead","approval_status_field":"Status","approved_value":"Working - Contacted","rejected_value":"Closed - Not Converted"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/165","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"Approval Process","source_difficulty":"hard","template_id":"admin_038","template_name":"configure_approval_process_actions","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/165"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/165"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","sha256":"ea1ed47b0ea57258c45f934f401208adc16e6fc941f1d519f9622c296af8d952","source_json_pointer":"/configure_approval_process_actions"}],"source_schema_notes":[],"template_registry":{"name":"configure_approval_process_actions","original":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"source_path":"scuba/task_generators/templates_info_admin_new.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_admin_new.json","sha256":"da69ee8d57c3223a1f13445619fbe2a0c3fe8a01d16dc42a9e611c29d9996e4d","source_json_pointer":"/configure_approval_process_actions","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nFor the Lead.required_lead_action approval process, make the end-of-process steps: on final approval, build a new action named Update Status to set the Status field to Working - Contacted; on final rejection, configure a new action named Close Lead to set Status to Closed - Not Converted.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.","memory_content_status":"example_text_present","original":{"category":"admin","subcategory":"Approval Process","task_id":"admin_038_005","query_templated":"Configure the Lead.required_lead_action approval process's final approval actions by adding a new action with the name Update Status, which set the Status field value to Working - Contacted. Similarly, add a new action under the final rejection actions with the name Close Lead, which set the field value to Closed - Not Converted.","query":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save.\n\n## Your ultimate task\n```\nFor the Lead.required_lead_action approval process, make the end-of-process steps: on final approval, build a new action named Update Status to set the Status field to Working - Contacted; on final rejection, configure a new action named Close Lead to set Status to Closed - Not Converted.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"approval_process_name":"Lead.required_lead_action","final_approval_action_name":"Update Status","final_rejection_action_name":"Close Lead","approval_status_field":"Status","approved_value":"Working - Contacted","rejected_value":"Closed - Not Converted"},"difficulty":"hard","query_template_name":"configure_approval_process_actions","query_template_metadata":{"template_id":"admin_038","template_string":"Configure the {approval_process_name} approval process's final approval actions by adding a new action with the name {final_approval_action_name}, which set the {approval_status_field} field value to {approved_value}. Similarly, add a new action under the final rejection actions with the name {final_rejection_action_name}, which set the field value to {rejected_value}.","prerequisites":{"metadata":{"types_and_members":{"ApprovalProcess":["Account.required_account_action","Campaign.required_campaign_action","Contact.required_contact_action","Lead.required_lead_action","Product2.required_product_action"]}}},"objects":[],"metadata_types":["ApprovalProcess"]},"instance_dict":{"approval_process_name":"Lead.required_lead_action","final_approval_action_name":"Update Status","final_rejection_action_name":"Close Lead","approval_status_field":"Status","approved_value":"Working - Contacted","rejected_value":"Closed - Not Converted"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Configure the required_account_action approval process's final approval actions by adding a new action with the name \"mark_clean\", which sets the \"Clean Status\" field value to \"In Sync\". Similarly, add a new action under the final rejection actions with the name \"mark_inactive\", which sets the \"Clean Status\" field value to \"Inactive\".\nEXPERIENCE:\nSuccessful plan to configure final approval/rejection actions for required_account_action:\n\n1) Access the approval process\n- Log in to Salesforce.\n- Open Setup.\n- Navigate: Process Automation > Approval Processes.\n- In “Manage Approval Processes For” dropdown, select Account.\n- Open the approval process named required_account_action.\n\n2) Add final approval field update\n- Click “Add New Final Approval Actions” > Field Update.\n- Name: mark_clean.\n- Field to Update: Clean Status.\n- New Field Value: In Sync (select a specific value).\n- Save.\n\n3) Add final rejection field update\n- Click “Add New Final Rejection Action” > Field Update.\n- Name: mark_inactive.\n- Field to Update: Clean Status.\n- New Field Value: Inactive (select a specific value).\n- Save.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:Configure the \"required_product_action\" approval process's final approval actions by adding a new action with the name \"Activate\", which sets the \"Active\" field value to \"True\". Similarly, add a new action under the \"final rejection actions\" with the name \"Deserialize\", which sets the Active field value to \"False\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- Navigate to Process Automation > Approval Processes.\n- In the Manage… dropdown, select Product and open the approval process “required_product_action”.\n\nConfigure final approval action:\n- Click Add New Final Approval Action > Field Update.\n- Set Name: Activate.\n- Set Field to Update: Active.\n- Set value to True.\n- Save.\n\nConfigure final rejection action:\n- Click Add New Final Rejection Action > Field Update.\n- Set Name: Deserialize.\n- Set Field to Update: Active.\n- Set value to False.\n- Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/165"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_admin_new.py#L240","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_admin_new.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_configure_approval_process_actions","data_retrieval_spec":[{"type":"metadata","id":"approval_process","params":{"types_and_members":{"ApprovalProcess":["*"]}}},{"type":"metadata","id":"workflows","params":{"types_and_members":{"Workflow":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"06d6b469eeb2a90034f59f5609c4436d5891a322914c728599912bf2d3cdaff1"} {"id":"scuba:admin_039_001","source_id":"scuba","original_id":"admin_039_001","kind":"benchmark_task","instruction":"Based on the org’s object settings, if a user only has the “Account Access” permission set, can they delete any Account records? Reply with “yes” or “no” only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_001","query_templated":"A user is assigned with Account Access permission set. Will the user be able to delet any records under the Account Object? Grounded on the salesforce org's object settings, Only answer with \"yes\" or \"no\".","query":"Based on the org’s object settings, if a user only has the “Account Access” permission set, can they delete any Account records? Reply with “yes” or “no” only.","human_review_comment":"accept","ground_truth_dict":{"answer":"no"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"A user is assigned with Account Access permission set. Will the user be able to delet any records under the Account Object? Grounded on the salesforce org's object settings, Only answer with \"yes\" or \"no\"."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/166","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"qa","source_difficulty":"medium","template_id":"admin_039","template_name":"qa_admin_039","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/166"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/166"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039"}],"source_schema_notes":[],"template_registry":{"name":"qa_admin_039","original":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nBased on the org’s object settings, if a user only has the “Account Access” permission set, can they delete any Account records? Reply with “yes” or “no” only.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_001","query_templated":"A user is assigned with Account Access permission set. Will the user be able to delet any records under the Account Object? Grounded on the salesforce org's object settings, Only answer with \"yes\" or \"no\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nBased on the org’s object settings, if a user only has the “Account Access” permission set, can they delete any Account records? Reply with “yes” or “no” only.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"no"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"A user is assigned with Account Access permission set. Will the user be able to delet any records under the Account Object? Grounded on the salesforce org's object settings, Only answer with \"yes\" or \"no\"."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/166"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"12a558574ac1902ed0804923ef5b357a0f5d2213790248b7ad7535d23e4c816d"} {"id":"scuba:admin_039_002","source_id":"scuba","original_id":"admin_039_002","kind":"benchmark_task","instruction":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_002","query_templated":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.","query":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Agentforce Service Agent Secure Base"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/167","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"qa","source_difficulty":"medium","template_id":"admin_039","template_name":"qa_admin_039","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/167"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/167"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039"}],"source_schema_notes":[],"template_registry":{"name":"qa_admin_039","original":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nThe Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_002","query_templated":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nThe Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Agentforce Service Agent Secure Base"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"The Case Object is associated with multiple permission sets. Among the [Agentforce Service Agent Secure Base, C360 High Scale Flow Integration User, Cases Admin, and ConnectivityServiceCASCPermSet] permission sets, which one does not have the Edit priviledge? Grounded on the salesforce org's object settings, answer the question with the precise name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/167"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"b2914a6f274961d0724eafb7e5879808dd953be867115e0da3c90690c3cf0033"} {"id":"scuba:admin_039_003","source_id":"scuba","original_id":"admin_039_003","kind":"benchmark_task","instruction":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_003","query_templated":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].","query":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Analytics Cloud Integration User"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/168","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"qa","source_difficulty":"medium","template_id":"admin_039","template_name":"qa_admin_039","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/168"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/168"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039"}],"source_schema_notes":[],"template_registry":{"name":"qa_admin_039","original":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nGrounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_003","query_templated":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nGrounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User].\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Analytics Cloud Integration User"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Grounded on the salesforce org's Opportunity object, which Profile has the most limited access. Choose one and reply with the exact name from the list [Custom: Sales Profile, Gold Partner User, Analytics Cloud Integration User]."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/168"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"685a54fc1cb65d692f0f445e6c43989f376d49b8f58d7cda740f0568ce10e5da"} {"id":"scuba:admin_039_004","source_id":"scuba","original_id":"admin_039_004","kind":"benchmark_task","instruction":"From the existing permission set groups, which one grants Delete on the Account object? Choose only one from Product Admin, Commerce_Shopper, or Trainee Analyst, and answer with the precise name based on the org’s object settings.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_004","query_templated":"Looking at the existing permission set groups under the Account Object, I would like to pick one that has the Delete priviledge. Could you recommend one out of the [Product Admin, Commerce_Shopper, and Trainee Analyst]? Grounded on the salesforce org's object settings, answer the question with the precise name.","query":"From the existing permission set groups, which one grants Delete on the Account object? Choose only one from Product Admin, Commerce_Shopper, or Trainee Analyst, and answer with the precise name based on the org’s object settings.","human_review_comment":"accept","ground_truth_dict":{"answer":"Product Admin"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Looking at the existing permission set groups under the Account Object, I would like to pick one that has the Delete priviledge. Could you recommend one out of the [Product Admin, Commerce_Shopper, and Trainee Analyst]? Grounded on the salesforce org's object settings, answer the question with the precise name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/169","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"qa","source_difficulty":"medium","template_id":"admin_039","template_name":"qa_admin_039","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/169"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/169"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039"}],"source_schema_notes":[],"template_registry":{"name":"qa_admin_039","original":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the existing permission set groups, which one grants Delete on the Account object? Choose only one from Product Admin, Commerce_Shopper, or Trainee Analyst, and answer with the precise name based on the org’s object settings.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_004","query_templated":"Looking at the existing permission set groups under the Account Object, I would like to pick one that has the Delete priviledge. Could you recommend one out of the [Product Admin, Commerce_Shopper, and Trainee Analyst]? Grounded on the salesforce org's object settings, answer the question with the precise name.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the existing permission set groups, which one grants Delete on the Account object? Choose only one from Product Admin, Commerce_Shopper, or Trainee Analyst, and answer with the precise name based on the org’s object settings.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Product Admin"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Looking at the existing permission set groups under the Account Object, I would like to pick one that has the Delete priviledge. Could you recommend one out of the [Product Admin, Commerce_Shopper, and Trainee Analyst]? Grounded on the salesforce org's object settings, answer the question with the precise name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/169"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"e8887b5d5a56ceb6feac2d33f476a1a0197c15f12c1560c6480844eb63b5d363"} {"id":"scuba:admin_039_005","source_id":"scuba","original_id":"admin_039_005","kind":"benchmark_task","instruction":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_005","query_templated":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.","query":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Order"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/170","source_record_type":"crm_workflow_instance","category":"admin","role":"admin","subcategory":"qa","source_difficulty":"medium","template_id":"admin_039","template_name":"qa_admin_039","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/170"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/170"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039"}],"source_schema_notes":[],"template_registry":{"name":"qa_admin_039","original":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_admin_039","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nInspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"admin","subcategory":"qa","task_id":"admin_039_005","query_templated":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nInspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"answer":"Order"},"difficulty":"medium","query_template_name":"qa_admin_039","query_template_metadata":{"template_id":"admin_039","template_string":"{information}","prerequisites":{},"objects":[],"metadata_types":[]},"instance_dict":{"information":"Inspect the Buyer permission set and compare its object permissions for the Account and Order Obejcts.Which one has the delete priviledge? Answer the question with the precise object API Name."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/170"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"d707dfab2f84a674296c57627a620bc3818832d3f7bd3fce6139861879fc4fef"} {"id":"scuba:sales_001_001","source_id":"scuba","original_id":"sales_001_001","kind":"benchmark_task","instruction":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_001","query_templated":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com","query":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com","human_review_comment":"reject; use the orginal","ground_truth_dict":{"company_name":"Old Balance","contact_name":"Marcus Roth","phone_number":"888-333-222","email_address":"marcus.roth@oldbalance.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company.","contact_details":"Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/171","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"medium","template_id":"sales_001","template_name":"create_account_and_contact","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/171"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/171"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_and_contact"}],"source_schema_notes":[],"template_registry":{"name":"create_account_and_contact","original":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_001","query_templated":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com","query":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"company_name":"Old Balance","contact_name":"Marcus Roth","phone_number":"888-333-222","email_address":"marcus.roth@oldbalance.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company.","contact_details":"Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/171"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L61","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_and_contact","data_retrieval_spec":[{"type":"soql","id":"get_company_account","params":{"query":"SELECT Id, Name FROM Account WHERE Name = '{company_name}'"}},{"type":"soql","id":"get_contact","params":{"query":"SELECT Id, Account.Name, Name, Email, Phone, MobilePhone, HomePhone FROM Contact WHERE Name = '{contact_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"33c66a8abce62380658d0920a946b2250eb432e1f54488b48af959b5ea2141c5"} {"id":"scuba:sales_001_002","source_id":"scuba","original_id":"sales_001_002","kind":"benchmark_task","instruction":"Quick follow-up from my chat with Sarah Kim (Director of Operations) at Next Horizon Technologies—definite potential here. Please: - Create an account for Next Horizon Technologies - Set Jordan Lee as the point of contact - Mobile: 777-444-1111","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_002","query_templated":"I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Jordan Lee, whose mobile number is 777-444-1111","query":"Quick follow-up from my chat with Sarah Kim (Director of Operations) at Next Horizon Technologies—definite potential here. Please: - Create an account for Next Horizon Technologies - Set Jordan Lee as the point of contact - Mobile: 777-444-1111","human_review_comment":"accept","ground_truth_dict":{"company_name":"Next Horizon Technologies","contact_name":"Jordan Lee","phone_number":"777-444-1111","email_address":""},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company.","contact_details":"Jordan Lee, whose mobile number is 777-444-1111"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/172","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"medium","template_id":"sales_001","template_name":"create_account_and_contact","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/172"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/172"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_and_contact"}],"source_schema_notes":[],"template_registry":{"name":"create_account_and_contact","original":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nQuick follow-up from my chat with Sarah Kim (Director of Operations) at Next Horizon Technologies—definite potential here. Please: - Create an account for Next Horizon Technologies - Set Jordan Lee as the point of contact - Mobile: 777-444-1111\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_002","query_templated":"I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Jordan Lee, whose mobile number is 777-444-1111","query":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nQuick follow-up from my chat with Sarah Kim (Director of Operations) at Next Horizon Technologies—definite potential here. Please: - Create an account for Next Horizon Technologies - Set Jordan Lee as the point of contact - Mobile: 777-444-1111\n```","human_review_comment":"accept","ground_truth_dict":{"company_name":"Next Horizon Technologies","contact_name":"Jordan Lee","phone_number":"777-444-1111","email_address":""},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company.","contact_details":"Jordan Lee, whose mobile number is 777-444-1111"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/172"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L61","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_and_contact","data_retrieval_spec":[{"type":"soql","id":"get_company_account","params":{"query":"SELECT Id, Name FROM Account WHERE Name = '{company_name}'"}},{"type":"soql","id":"get_contact","params":{"query":"SELECT Id, Account.Name, Name, Email, Phone, MobilePhone, HomePhone FROM Contact WHERE Name = '{contact_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"bfeb6c8b8b5bc288eb420d64456e63975f965703647b37cfc9f7f427601de7bc"} {"id":"scuba:sales_001_003","source_id":"scuba","original_id":"sales_001_003","kind":"benchmark_task","instruction":"Could you get GreenTrail Logistics set up in the CRM? I spoke with Daniel Ortiz (SVP of Partnerships) and there’s real upside. Make Emily Zhao the main point of contact and record her email as emily.zhao@greentrail.com.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_003","query_templated":"I just met Daniel Ortiz, the SVP of Partnerships from GreenTrail Logistics, and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Emily Zhao, whose email is emily.zhao@greentrail.com","query":"Could you get GreenTrail Logistics set up in the CRM? I spoke with Daniel Ortiz (SVP of Partnerships) and there’s real upside. Make Emily Zhao the main point of contact and record her email as emily.zhao@greentrail.com.","human_review_comment":"accept","ground_truth_dict":{"company_name":"GreenTrail Logistics","contact_name":"Emily Zhao","phone_number":"","email_address":"emily.zhao@greentrail.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met Daniel Ortiz, the SVP of Partnerships from GreenTrail Logistics, and there's a potential to start a business with this company.","contact_details":"Emily Zhao, whose email is emily.zhao@greentrail.com"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/173","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"medium","template_id":"sales_001","template_name":"create_account_and_contact","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/173"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/173"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_and_contact"}],"source_schema_notes":[],"template_registry":{"name":"create_account_and_contact","original":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCould you get GreenTrail Logistics set up in the CRM? I spoke with Daniel Ortiz (SVP of Partnerships) and there’s real upside. Make Emily Zhao the main point of contact and record her email as emily.zhao@greentrail.com.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_003","query_templated":"I just met Daniel Ortiz, the SVP of Partnerships from GreenTrail Logistics, and there's a potential to start a business with this company.. Help me to create an account for this company and the point of contact should be Emily Zhao, whose email is emily.zhao@greentrail.com","query":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCould you get GreenTrail Logistics set up in the CRM? I spoke with Daniel Ortiz (SVP of Partnerships) and there’s real upside. Make Emily Zhao the main point of contact and record her email as emily.zhao@greentrail.com.\n```","human_review_comment":"accept","ground_truth_dict":{"company_name":"GreenTrail Logistics","contact_name":"Emily Zhao","phone_number":"","email_address":"emily.zhao@greentrail.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just met Daniel Ortiz, the SVP of Partnerships from GreenTrail Logistics, and there's a potential to start a business with this company.","contact_details":"Emily Zhao, whose email is emily.zhao@greentrail.com"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/173"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L61","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_and_contact","data_retrieval_spec":[{"type":"soql","id":"get_company_account","params":{"query":"SELECT Id, Name FROM Account WHERE Name = '{company_name}'"}},{"type":"soql","id":"get_contact","params":{"query":"SELECT Id, Account.Name, Name, Email, Phone, MobilePhone, HomePhone FROM Contact WHERE Name = '{contact_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9d5dad202aede7d2e28e3525954c3098f8cf7b40a6ef16afbc81c729284b32ea"} {"id":"scuba:sales_001_004","source_id":"scuba","original_id":"sales_001_004","kind":"benchmark_task","instruction":"Today’s call with TechFlow Solutions’ procurement team went great—they’re planning to scale their cloud infrastructure and line up well with our enterprise package. Please open a company account for TechFlow Solutions and list Sarah Mitchell (IT Director) as the POC. You can reach her at (555) 123-4567 or sarah.mitchell@techflowsolutions.com.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_004","query_templated":"I just had a productive call with the procurement team at TechFlow Solutions. They're looking to expand their cloud infrastructure services and seem like a great fit for our enterprise package.. Help me to create an account for this company and the point of contact should be Sarah Mitchell, their IT Director, who can be reached at (555) 123-4567 or sarah.mitchell@techflowsolutions.com","query":"Today’s call with TechFlow Solutions’ procurement team went great—they’re planning to scale their cloud infrastructure and line up well with our enterprise package. Please open a company account for TechFlow Solutions and list Sarah Mitchell (IT Director) as the POC. You can reach her at (555) 123-4567 or sarah.mitchell@techflowsolutions.com.","human_review_comment":"accept","ground_truth_dict":{"company_name":"TechFlow Solutions","contact_name":"Sarah Mitchell","phone_number":"(555) 123-4567","email_address":"sarah.mitchell@techflowsolutions.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just had a productive call with the procurement team at TechFlow Solutions. They're looking to expand their cloud infrastructure services and seem like a great fit for our enterprise package.","contact_details":"Sarah Mitchell, their IT Director, who can be reached at (555) 123-4567 or sarah.mitchell@techflowsolutions.com"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/174","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"medium","template_id":"sales_001","template_name":"create_account_and_contact","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/174"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/174"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_and_contact"}],"source_schema_notes":[],"template_registry":{"name":"create_account_and_contact","original":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nToday’s call with TechFlow Solutions’ procurement team went great—they’re planning to scale their cloud infrastructure and line up well with our enterprise package. Please open a company account for TechFlow Solutions and list Sarah Mitchell (IT Director) as the POC. You can reach her at (555) 123-4567 or sarah.mitchell@techflowsolutions.com.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_004","query_templated":"I just had a productive call with the procurement team at TechFlow Solutions. They're looking to expand their cloud infrastructure services and seem like a great fit for our enterprise package.. Help me to create an account for this company and the point of contact should be Sarah Mitchell, their IT Director, who can be reached at (555) 123-4567 or sarah.mitchell@techflowsolutions.com","query":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nToday’s call with TechFlow Solutions’ procurement team went great—they’re planning to scale their cloud infrastructure and line up well with our enterprise package. Please open a company account for TechFlow Solutions and list Sarah Mitchell (IT Director) as the POC. You can reach her at (555) 123-4567 or sarah.mitchell@techflowsolutions.com.\n```","human_review_comment":"accept","ground_truth_dict":{"company_name":"TechFlow Solutions","contact_name":"Sarah Mitchell","phone_number":"(555) 123-4567","email_address":"sarah.mitchell@techflowsolutions.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"I just had a productive call with the procurement team at TechFlow Solutions. They're looking to expand their cloud infrastructure services and seem like a great fit for our enterprise package.","contact_details":"Sarah Mitchell, their IT Director, who can be reached at (555) 123-4567 or sarah.mitchell@techflowsolutions.com"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/174"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L61","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_and_contact","data_retrieval_spec":[{"type":"soql","id":"get_company_account","params":{"query":"SELECT Id, Name FROM Account WHERE Name = '{company_name}'"}},{"type":"soql","id":"get_contact","params":{"query":"SELECT Id, Account.Name, Name, Email, Phone, MobilePhone, HomePhone FROM Contact WHERE Name = '{contact_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"61b699c969f90fdfdd864650d16b77145781810d3fda5ca2d7313654c43e9ac9"} {"id":"scuba:sales_001_005","source_id":"scuba","original_id":"sales_001_005","kind":"benchmark_task","instruction":"After connecting with the leadership team at Global Manufacturing Corp at yesterday’s industry conference, I’m confident they’re a fit for our supply chain optimization solutions. Please create an account for Global Manufacturing Corp and set Michael Chen, VP of Operations, as the point of contact. He prefers calls at (555) 987-6543; if he’s unavailable, use m.chen@globalmanufacturing.com.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_005","query_templated":"After meeting with the leadership team at Global Manufacturing Corp during yesterday's industry conference, I believe they would be an excellent prospect for our supply chain optimization solutions.. Help me to create an account for this company and the point of contact should be Michael Chen, VP of Operations, who mentioned he's available at (555) 987-6543. If not, he should be reached at m.chen@globalmanufacturing.com","query":"After connecting with the leadership team at Global Manufacturing Corp at yesterday’s industry conference, I’m confident they’re a fit for our supply chain optimization solutions. Please create an account for Global Manufacturing Corp and set Michael Chen, VP of Operations, as the point of contact. He prefers calls at (555) 987-6543; if he’s unavailable, use m.chen@globalmanufacturing.com.","human_review_comment":"accept","ground_truth_dict":{"company_name":"Global Manufacturing Corp","contact_name":"Michael Chen","phone_number":"(555) 987-6543","email_address":"m.chen@globalmanufacturing.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"After meeting with the leadership team at Global Manufacturing Corp during yesterday's industry conference, I believe they would be an excellent prospect for our supply chain optimization solutions.","contact_details":"Michael Chen, VP of Operations, who mentioned he's available at (555) 987-6543. If not, he should be reached at m.chen@globalmanufacturing.com"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/175","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"medium","template_id":"sales_001","template_name":"create_account_and_contact","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/175"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/175"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_and_contact"}],"source_schema_notes":[],"template_registry":{"name":"create_account_and_contact","original":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_and_contact","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nAfter connecting with the leadership team at Global Manufacturing Corp at yesterday’s industry conference, I’m confident they’re a fit for our supply chain optimization solutions. Please create an account for Global Manufacturing Corp and set Michael Chen, VP of Operations, as the point of contact. He prefers calls at (555) 987-6543; if he’s unavailable, use m.chen@globalmanufacturing.com.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_001_005","query_templated":"After meeting with the leadership team at Global Manufacturing Corp during yesterday's industry conference, I believe they would be an excellent prospect for our supply chain optimization solutions.. Help me to create an account for this company and the point of contact should be Michael Chen, VP of Operations, who mentioned he's available at (555) 987-6543. If not, he should be reached at m.chen@globalmanufacturing.com","query":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nAfter connecting with the leadership team at Global Manufacturing Corp at yesterday’s industry conference, I’m confident they’re a fit for our supply chain optimization solutions. Please create an account for Global Manufacturing Corp and set Michael Chen, VP of Operations, as the point of contact. He prefers calls at (555) 987-6543; if he’s unavailable, use m.chen@globalmanufacturing.com.\n```","human_review_comment":"accept","ground_truth_dict":{"company_name":"Global Manufacturing Corp","contact_name":"Michael Chen","phone_number":"(555) 987-6543","email_address":"m.chen@globalmanufacturing.com"},"difficulty":"medium","query_template_name":"create_account_and_contact","query_template_metadata":{"template_id":"sales_001","template_string":"{Sentence that reveals the company name}. Help me to create an account for this company and the point of contact should be {contact_details}","prerequisites":{},"objects":["Contact","Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"Sentence that reveals the company name":"After meeting with the leadership team at Global Manufacturing Corp during yesterday's industry conference, I believe they would be an excellent prospect for our supply chain optimization solutions.","contact_details":"Michael Chen, VP of Operations, who mentioned he's available at (555) 987-6543. If not, he should be reached at m.chen@globalmanufacturing.com"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I just met the Leo Luis, the VP from the Old Balance and there's a potential to start a business with this company. Help me to create an account for this company and the point of contact should be Marcus Roth, whose mobile number is 888-333-222 and email is marcus.roth@oldbalance.com\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Open the App Launcher, search for “accounts,” and open the Accounts app/tab.\n- Click New to create an account.\n- Enter Account Name: Old Balance and save.\n- From the newly created account, click New Contact.\n- Enter:\n - First Name: Marcus\n - Last Name: Roth\n - Mobile: 888-333-222\n - Email: marcus.roth@oldbalance.com\n- Save the contact to set Marcus Roth as the point of contact for the Old Balance account.\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:I just met Sarah Kim, the Director of Operations from Next Horizon Technologies, and there's a potential to start a business with this company. Help me to create an account for this company. The point of contact should be Jordan Lee, whose mobile number is 777-444-1111.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Accounts and select New.\n- Enter Account Name: “Next Horizon Technologies” and Save.\n- From the account, select New Contact.\n- Enter:\n - First Name: Jordan\n - Last Name: Lee\n - Mobile: 777-444-1111\n - Save the contact.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/175"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L61","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_and_contact","data_retrieval_spec":[{"type":"soql","id":"get_company_account","params":{"query":"SELECT Id, Name FROM Account WHERE Name = '{company_name}'"}},{"type":"soql","id":"get_contact","params":{"query":"SELECT Id, Account.Name, Name, Email, Phone, MobilePhone, HomePhone FROM Contact WHERE Name = '{contact_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e6e86c7d7d4d34966b62746e81ad04931e221984ef57f3ec2535aaf4cf3e2836"} {"id":"scuba:sales_002_001","source_id":"scuba","original_id":"sales_002_001","kind":"benchmark_task","instruction":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_001","query_templated":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry).. Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe..","query":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"LyLyApple - HQ":["LyLyPear - Asia","LyLyCherry - Europe"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry).","setup_instructions":"Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/176","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"easy","template_id":"sales_002","template_name":"create_account_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/176"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/176"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"create_account_hierarchy","original":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_001","query_templated":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry).. Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe..","query":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"LyLyApple - HQ":["LyLyPear - Asia","LyLyCherry - Europe"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry).","setup_instructions":"Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/176"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L161","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_hierarchy","data_retrieval_spec":[{"type":"soql","id":"accounts_info","params":{"query":"SELECT Id, Name, ParentId FROM Account"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"40402c760e7108644c071aaa9093184582f592ae7cf3b431b53ae78cd8c10e65"} {"id":"scuba:sales_002_002","source_id":"scuba","original_id":"sales_002_002","kind":"benchmark_task","instruction":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_002","query_templated":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..","query":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"NovaAxis Corp – Global":["NovaAxis Seoul – South Korea"],"TerraLink Systems – HQ":["TerraLink Nairobi – Kenya"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).","setup_instructions":"Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/177","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"easy","template_id":"sales_002","template_name":"create_account_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/177"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/177"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"create_account_hierarchy","original":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nBoth NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_002","query_templated":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..","query":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nBoth NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).. Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya..\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"NovaAxis Corp – Global":["NovaAxis Seoul – South Korea"],"TerraLink Systems – HQ":["TerraLink Nairobi – Kenya"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi).","setup_instructions":"Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ, and add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/177"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L161","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_hierarchy","data_retrieval_spec":[{"type":"soql","id":"accounts_info","params":{"query":"SELECT Id, Name, ParentId FROM Account"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"ffc7a07b6a19c3a04411394870a5357623c8cd828aba4c6c6653611ce61e0cec"} {"id":"scuba:sales_002_003","source_id":"scuba","original_id":"sales_002_003","kind":"benchmark_task","instruction":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_003","query_templated":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches.. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany..","query":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"CyberLoom Inc. – HQ":["CyberLoom Berlin – Germany"],"Vertex Dynamics – Global":[]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches.","setup_instructions":"Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/178","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"easy","template_id":"sales_002","template_name":"create_account_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/178"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/178"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"create_account_hierarchy","original":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_003","query_templated":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches.. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany..","query":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches. Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"CyberLoom Inc. – HQ":["CyberLoom Berlin – Germany"],"Vertex Dynamics – Global":[]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"CyberLoom Inc. and Vertex Dynamics are expanding in different ways. CyberLoom has launched a new research center in Germany (CyberLoom Berlin), while Vertex Dynamics has not yet opened any new branches.","setup_instructions":"Create two parent accounts: CyberLoom Inc. – HQ and Vertex Dynamics – Global, and add a subordinate account only under CyberLoom: CyberLoom Berlin – Germany."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/178"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L161","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_hierarchy","data_retrieval_spec":[{"type":"soql","id":"accounts_info","params":{"query":"SELECT Id, Name, ParentId FROM Account"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a29f625dbf88054e01fd5cf8865e850c438b59ad1ab9c945973b801278e70799"} {"id":"scuba:sales_002_004","source_id":"scuba","original_id":"sales_002_004","kind":"benchmark_task","instruction":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_004","query_templated":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..","query":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"CloudWave Solutions - HQ":["CloudWave NA - North America","CloudWave AU - Australia"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).","setup_instructions":"Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/179","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"easy","template_id":"sales_002","template_name":"create_account_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/179"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/179"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"create_account_hierarchy","original":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_004","query_templated":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..","query":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).. Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia..\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"CloudWave Solutions - HQ":["CloudWave NA - North America","CloudWave AU - Australia"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"The company CloudWave Solutions opens two new sites in North America (CloudWave NA) and Australia (CloudWave AU).","setup_instructions":"Create an account with name CloudWave Solutions - HQ and two subordinates account CloudWave NA - North America, CloudWave AU - Australia."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/179"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L161","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_hierarchy","data_retrieval_spec":[{"type":"soql","id":"accounts_info","params":{"query":"SELECT Id, Name, ParentId FROM Account"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"51875cf93d90d1a8ff072e548032d590863df138fc6fa0e2ef0938322fdf2994"} {"id":"scuba:sales_002_005","source_id":"scuba","original_id":"sales_002_005","kind":"benchmark_task","instruction":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_005","query_templated":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..","query":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"MegaCorp Financial - Parent":["DataSecure Solutions - Cybersecurity Division","GreenTech Innovations - Renewable Energy Division"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.","setup_instructions":"Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/180","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Account & Contacts","source_difficulty":"easy","template_id":"sales_002","template_name":"create_account_hierarchy","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/180"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/180"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_account_hierarchy"}],"source_schema_notes":[],"template_registry":{"name":"create_account_hierarchy","original":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_account_hierarchy","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nMegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Account & Contacts","task_id":"sales_002_005","query_templated":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..","query":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used.\n\n## Your ultimate task\n```\nMegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.. Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division..\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"hierarchies":{"MegaCorp Financial - Parent":["DataSecure Solutions - Cybersecurity Division","GreenTech Innovations - Renewable Energy Division"]}},"difficulty":"easy","query_template_name":"create_account_hierarchy","query_template_metadata":{"template_id":"sales_002","template_string":"{company_hierarchy_description}. {setup_instructions}.","prerequisites":{},"objects":["Account"],"metadata_types":["ValidationRule"]},"instance_dict":{"company_hierarchy_description":"MegaCorp Financial acquired two specialized firms: DataSecure Solutions (cybersecurity) and GreenTech Innovations (renewable energy consulting). They want to maintain the acquired companies as separate divisions under the MegaCorp umbrella.","setup_instructions":"Create an account with name MegaCorp Financial - Parent and two subsidiary accounts DataSecure Solutions - Cybersecurity Division, GreenTech Innovations - Renewable Energy Division."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The company LyLyApple opens two new sites in Asia (LyLyPear) and Europe (LyLyCherry). Create an account with the name LyLyApple – HQ and two subordinate accounts: LyLyPear – Asia and LyLyCherry – Europe.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the application.\n- Navigate to Accounts and click New.\n- Create the parent account:\n - Enter Account Name: “LyLyApple - HQ”.\n - Click Save & New to proceed to creating another account.\n- Create the first child account:\n - Enter Account Name: “LyLyPear - Asia”.\n - Set Parent Account by searching/selecting “LyLyApple - HQ”.\n - Click Save & New.\n- Create the second child account:\n - Enter Account Name: “LyLyCherry - Europe”.\n - Set Parent Account to “LyLyApple - HQ”.\n - Click Save.\n- Verify hierarchy:\n - Go to Accounts, open “LyLyApple - HQ”, and select View Account Hierarchy.\nSample 2:\nTASK:Both NovaAxis Corp and TerraLink Systems have recently expanded their global footprint. NovaAxis has launched a new division in South Korea (NovaAxis Seoul), while TerraLink has opened a new site in Kenya (TerraLink Nairobi)]. [Create two separate parent accounts: NovaAxis Corp – Global and TerraLink Systems – HQ], and [add one subordinate account under each: NovaAxis Seoul – South Korea and TerraLink Nairobi – Kenya\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the app.\n- Navigate to Accounts and click New.\n- Create parent account 1:\n - Enter Account Name: \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create child account under parent 1:\n - Enter Account Name: \"NovaAxis Seoul - South Korea\".\n - Open the Parent Account lookup/search and select \"NovaAxis Corp - Global\".\n - Click \"Save & New\".\n- Create parent account 2:\n - Enter Account Name: \"TerraLink Systems - HQ\".\n - Click \"Save & New\".\n- Create child account under parent 2:\n - Enter Account Name: \"TerraLink Nairobi - Kenya\".\n - Open the Parent Account lookup/search and select \"TerraLink Systems - HQ\".\n - Click \"Save\".\n\nOptional verification:\n- Open Accounts, select each parent (\"NovaAxis Corp - Global\" and \"TerraLink Systems - HQ\"), and use \"View Account Hierarchy\" to confirm the child relationships.\n\nNotes:\n- Used the \"Save & New\" button to streamline consecutive record creation.\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/180"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L161","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_account_hierarchy","data_retrieval_spec":[{"type":"soql","id":"accounts_info","params":{"query":"SELECT Id, Name, ParentId FROM Account"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"de3b5c05194fced9e78e5273ef914d27416641c143dce1fd292bc0901bb81830"} {"id":"scuba:sales_003_001","source_id":"scuba","original_id":"sales_003_001","kind":"benchmark_task","instruction":"Could you log a web-sourced lead for John Doe at sForce company? I just spoke with him; please set his attitude to warm and the industry to Technology.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_001","query_templated":"I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.","query":"Could you log a web-sourced lead for John Doe at sForce company? I just spoke with him; please set his attitude to warm and the industry to Technology.","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"lead_name":"John Doe","lead_source":"Web","rating":"Warm","company_name":"sForce"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/181","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_003","template_name":"create_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/181"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/181"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_lead"}],"source_schema_notes":[],"template_registry":{"name":"create_lead","original":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCould you log a web-sourced lead for John Doe at sForce company? I just spoke with him; please set his attitude to warm and the industry to Technology.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_001","query_templated":"I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.","query":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCould you log a web-sourced lead for John Doe at sForce company? I just spoke with him; please set his attitude to warm and the industry to Technology.\n```","human_review_comment":"tweak a bit to avoid ambiguity","ground_truth_dict":{"lead_name":"John Doe","lead_source":"Web","rating":"Warm","company_name":"sForce"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/181"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L190","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, LeadSource, Name, Company, Rating FROM Lead WHERE Name='{lead_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"545f66d7749e04ce8e237f4c355a0fea3fa1565acadae1a12e8f38ca093dcdc2"} {"id":"scuba:sales_003_002","source_id":"scuba","original_id":"sales_003_002","kind":"benchmark_task","instruction":"Partner referral just came in: Maya Patel from GenePoint. I connected with her already—rate her attitude as hot and tag the industry as Biotechnology.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_002","query_templated":"I get a new lead source from partner referral. The lead name is Maya Patel from GenePoint company. I just contacted her and I would rate her attitude as hot. The industry is Biotechnology.","query":"Partner referral just came in: Maya Patel from GenePoint. I connected with her already—rate her attitude as hot and tag the industry as Biotechnology.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Maya Patel","lead_source":"Partner Referral","rating":"Hot","company_name":"GenePoint"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I get a new lead source from partner referral. The lead name is Maya Patel from GenePoint company. I just contacted her and I would rate her attitude as hot. The industry is Biotechnology."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/182","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_003","template_name":"create_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/182"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/182"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_lead"}],"source_schema_notes":[],"template_registry":{"name":"create_lead","original":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPartner referral just came in: Maya Patel from GenePoint. I connected with her already—rate her attitude as hot and tag the industry as Biotechnology.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_002","query_templated":"I get a new lead source from partner referral. The lead name is Maya Patel from GenePoint company. I just contacted her and I would rate her attitude as hot. The industry is Biotechnology.","query":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPartner referral just came in: Maya Patel from GenePoint. I connected with her already—rate her attitude as hot and tag the industry as Biotechnology.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Maya Patel","lead_source":"Partner Referral","rating":"Hot","company_name":"GenePoint"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I get a new lead source from partner referral. The lead name is Maya Patel from GenePoint company. I just contacted her and I would rate her attitude as hot. The industry is Biotechnology."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/182"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L190","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, LeadSource, Name, Company, Rating FROM Lead WHERE Name='{lead_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8243eb7c83b618b6c980acb9738953b8718f29bc84556d87ed8e0347f86f7ca5"} {"id":"scuba:sales_003_003","source_id":"scuba","original_id":"sales_003_003","kind":"benchmark_task","instruction":"New lead (source: Other) — Kevin Tran at HealthSync Solutions. Initial outreach done; attitude came across as cold. Industry should be Healthcare.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_003","query_templated":"I got a new lead source from Other. The lead name is Kevin Tran from HealthSync Solutions company. I just contacted him and I would rate his attitude as cold. The industry is Healthcare.","query":"New lead (source: Other) — Kevin Tran at HealthSync Solutions. Initial outreach done; attitude came across as cold. Industry should be Healthcare.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Kevin Tran","lead_source":"Other","rating":"Cold","company_name":"HealthSync Solutions"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I got a new lead source from Other. The lead name is Kevin Tran from HealthSync Solutions company. I just contacted him and I would rate his attitude as cold. The industry is Healthcare."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/183","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_003","template_name":"create_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/183"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/183"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_lead"}],"source_schema_notes":[],"template_registry":{"name":"create_lead","original":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nNew lead (source: Other) — Kevin Tran at HealthSync Solutions. Initial outreach done; attitude came across as cold. Industry should be Healthcare.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_003","query_templated":"I got a new lead source from Other. The lead name is Kevin Tran from HealthSync Solutions company. I just contacted him and I would rate his attitude as cold. The industry is Healthcare.","query":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nNew lead (source: Other) — Kevin Tran at HealthSync Solutions. Initial outreach done; attitude came across as cold. Industry should be Healthcare.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Kevin Tran","lead_source":"Other","rating":"Cold","company_name":"HealthSync Solutions"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"I got a new lead source from Other. The lead name is Kevin Tran from HealthSync Solutions company. I just contacted him and I would rate his attitude as cold. The industry is Healthcare."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/183"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L190","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, LeadSource, Name, Company, Rating FROM Lead WHERE Name='{lead_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"05a64baf79eb65b2dba7d096cea142ac4f6f7fbe99918a9bfb505835446d1a20"} {"id":"scuba:sales_003_004","source_id":"scuba","original_id":"sales_003_004","kind":"benchmark_task","instruction":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_004","query_templated":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.","query":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"lead_name":"David Rodriguez","lead_source":"Other","rating":"Warm","company_name":"Pinnacle Systems"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/184","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_003","template_name":"create_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/184"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/184"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_lead"}],"source_schema_notes":[],"template_registry":{"name":"create_lead","original":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nMet an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_004","query_templated":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.","query":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nMet an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"lead_name":"David Rodriguez","lead_source":"Other","rating":"Warm","company_name":"Pinnacle Systems"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"Met an interesting prospect at the trade show yesterday - David Rodriguez from Pinnacle Systems. He mentioned they're evaluating new vendors for their upcoming expansion and seemed very engaged during our conversation."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/184"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L190","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, LeadSource, Name, Company, Rating FROM Lead WHERE Name='{lead_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b133a00523df713def904bf0b9825d26840869a44cee3ade49cbaa3e7008078d"} {"id":"scuba:sales_003_005","source_id":"scuba","original_id":"sales_003_005","kind":"benchmark_task","instruction":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_005","query_templated":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.","query":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"lead_name":"Marcus Thompson","lead_source":"Other","rating":"Hot","company_name":"FutureTech Dynamics"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/185","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_003","template_name":"create_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/185"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/185"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_lead"}],"source_schema_notes":[],"template_registry":{"name":"create_lead","original":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nReceived a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_003_005","query_templated":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.","query":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nReceived a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"lead_name":"Marcus Thompson","lead_source":"Other","rating":"Hot","company_name":"FutureTech Dynamics"},"difficulty":"medium","query_template_name":"create_lead","query_template_metadata":{"template_id":"sales_003","template_string":"{free_form_sentence_with_lead_description}","prerequisites":{},"objects":["Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"free_form_sentence_with_lead_description":"Received a cold email inquiry from Marcus Thompson at FutureTech Dynamics. He expressed strong interest in our solutions for their digital transformation project and requested a follow-up call next week."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I get a new lead source from web. The lead name is John Doe from sForce company. I just contacted him and I would rate his attitude as warm. The industry is Technology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Leads and click New to create a lead.\n- Enter fields:\n - First Name: John\n - Last Name: Doe\n - Company: sForce\n- Set picklists:\n - Lead Source: Web\n - Rating: Warm\n - Industry: Technology (scroll within the dropdown to locate and select Technology)\n- Click Save to create the lead.\nSample 2:\nTASK:I received a new lead from a partner referral. The lead’s name is Maya Patel, representing the company GenePoint. I just reached out to her and would classify her attitude as hot. The industry for this lead is Biotechnology.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to the Leads section.\n- Click New to create a lead.\n- Enter fields:\n - First Name: Maya\n - Last Name: Patel\n - Company: GenePoint\n- Set picklist fields:\n - Lead Source: Partner Referral\n - Industry: Biotechnology\n - Rating: Hot\n- Click Save to create the lead.\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/185"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L190","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, LeadSource, Name, Company, Rating FROM Lead WHERE Name='{lead_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"206eefb938f2a7befd44407dbdcfaded817ac440d9f1f5eab0eea47c296d064f"} {"id":"scuba:sales_004_001","source_id":"scuba","original_id":"sales_004_001","kind":"benchmark_task","instruction":"Please convert Frank Knight’s lead and mark it as 'Closed - Converted'. If his company already exists as an Account, link the conversion to that Account; if it doesn’t, create a new Account using the lead’s company name. Then create a new Opportunity named Knight LLC Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_001","query_templated":"Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity.","query":"Please convert Frank Knight’s lead and mark it as 'Closed - Converted'. If his company already exists as an Account, link the conversion to that Account; if it doesn’t, create a new Account using the lead’s company name. Then create a new Opportunity named Knight LLC Opportunity.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Frank Knight","opportunity_name":"Knight LLC Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Frank Knight","opportunity_name":"Knight LLC Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/186","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_004","template_name":"convert_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/186"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/186"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/convert_lead"}],"source_schema_notes":[],"template_registry":{"name":"convert_lead","original":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nPlease convert Frank Knight’s lead and mark it as 'Closed - Converted'. If his company already exists as an Account, link the conversion to that Account; if it doesn’t, create a new Account using the lead’s company name. Then create a new Opportunity named Knight LLC Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_001","query_templated":"Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nPlease convert Frank Knight’s lead and mark it as 'Closed - Converted'. If his company already exists as an Account, link the conversion to that Account; if it doesn’t, create a new Account using the lead’s company name. Then create a new Opportunity named Knight LLC Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Frank Knight","opportunity_name":"Knight LLC Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Frank Knight","opportunity_name":"Knight LLC Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/186"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L313","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_convert_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, IsConverted, ConvertedAccountId, ConvertedOpportunityId FROM Lead WHERE Name='{lead_name}'"}},{"type":"soql","id":"check_account","params":{"query":"SELECT Id, Name FROM Account WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedAccountId']}'"}},{"type":"soql","id":"opportunity_check","params":{"query":"SELECT Id, Name FROM Opportunity WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedOpportunityId']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"cbb8bc19bf89520ac7cdbfa9d320fee6c4396fb5d1e4288c0dbaf20fddbecbdf"} {"id":"scuba:sales_004_002","source_id":"scuba","original_id":"sales_004_002","kind":"benchmark_task","instruction":"Step 1: Convert Jane Smith’s lead so the status is Closed - Converted. Step 2: During conversion, attach it to the existing Account if her company matches; if there’s no match, create a new Account with the lead’s company name. Step 3: Create an Opportunity titled Tech Giants Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_002","query_templated":"Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.","query":"Step 1: Convert Jane Smith’s lead so the status is Closed - Converted. Step 2: During conversion, attach it to the existing Account if her company matches; if there’s no match, create a new Account with the lead’s company name. Step 3: Create an Opportunity titled Tech Giants Opportunity.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Jane Smith","opportunity_name":"Tech Giants Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Jane Smith","opportunity_name":"Tech Giants Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/187","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_004","template_name":"convert_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/187"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/187"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/convert_lead"}],"source_schema_notes":[],"template_registry":{"name":"convert_lead","original":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nStep 1: Convert Jane Smith’s lead so the status is Closed - Converted. Step 2: During conversion, attach it to the existing Account if her company matches; if there’s no match, create a new Account with the lead’s company name. Step 3: Create an Opportunity titled Tech Giants Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_002","query_templated":"Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nStep 1: Convert Jane Smith’s lead so the status is Closed - Converted. Step 2: During conversion, attach it to the existing Account if her company matches; if there’s no match, create a new Account with the lead’s company name. Step 3: Create an Opportunity titled Tech Giants Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Jane Smith","opportunity_name":"Tech Giants Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Jane Smith","opportunity_name":"Tech Giants Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/187"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L313","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_convert_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, IsConverted, ConvertedAccountId, ConvertedOpportunityId FROM Lead WHERE Name='{lead_name}'"}},{"type":"soql","id":"check_account","params":{"query":"SELECT Id, Name FROM Account WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedAccountId']}'"}},{"type":"soql","id":"opportunity_check","params":{"query":"SELECT Id, Name FROM Opportunity WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedOpportunityId']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5819c2b05aa1752500701f678dd12b18039a9bbee087521bb6576716a007df59"} {"id":"scuba:sales_004_003","source_id":"scuba","original_id":"sales_004_003","kind":"benchmark_task","instruction":"Convert the lead for Alice Johnson to 'Closed - Converted'. If the company on her lead corresponds to an existing Account, select that Account; otherwise, create a new Account named exactly as the lead’s company. After that, set up a new Opportunity called Innovate Ltd Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_003","query_templated":"Convert the lead with name Alice Johnson to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Innovate Ltd Opportunity.","query":"Convert the lead for Alice Johnson to 'Closed - Converted'. If the company on her lead corresponds to an existing Account, select that Account; otherwise, create a new Account named exactly as the lead’s company. After that, set up a new Opportunity called Innovate Ltd Opportunity.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Alice Johnson","opportunity_name":"Innovate Ltd Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Alice Johnson","opportunity_name":"Innovate Ltd Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/188","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_004","template_name":"convert_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/188"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/188"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/convert_lead"}],"source_schema_notes":[],"template_registry":{"name":"convert_lead","original":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nConvert the lead for Alice Johnson to 'Closed - Converted'. If the company on her lead corresponds to an existing Account, select that Account; otherwise, create a new Account named exactly as the lead’s company. After that, set up a new Opportunity called Innovate Ltd Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_003","query_templated":"Convert the lead with name Alice Johnson to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Innovate Ltd Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nConvert the lead for Alice Johnson to 'Closed - Converted'. If the company on her lead corresponds to an existing Account, select that Account; otherwise, create a new Account named exactly as the lead’s company. After that, set up a new Opportunity called Innovate Ltd Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Alice Johnson","opportunity_name":"Innovate Ltd Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Alice Johnson","opportunity_name":"Innovate Ltd Opportunity"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/188"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L313","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_convert_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, IsConverted, ConvertedAccountId, ConvertedOpportunityId FROM Lead WHERE Name='{lead_name}'"}},{"type":"soql","id":"check_account","params":{"query":"SELECT Id, Name FROM Account WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedAccountId']}'"}},{"type":"soql","id":"opportunity_check","params":{"query":"SELECT Id, Name FROM Opportunity WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedOpportunityId']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8b132e2d5ada082630f561cbc858d5a26c71a1f758907854e66c70f7e4bfe27f"} {"id":"scuba:sales_004_004","source_id":"scuba","original_id":"sales_004_004","kind":"benchmark_task","instruction":"For David Lee — complete the lead conversion (status should be 'Closed - Converted'). Account handling: if the lead’s company matches an existing Account, use it; if not, spin up a new Account using the company name from the lead. Then open an Opportunity named Lee Services Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_004","query_templated":"Convert the lead with name David Lee to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Lee Services Opportunity.","query":"For David Lee — complete the lead conversion (status should be 'Closed - Converted'). Account handling: if the lead’s company matches an existing Account, use it; if not, spin up a new Account using the company name from the lead. Then open an Opportunity named Lee Services Opportunity.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"David Lee","opportunity_name":"Lee Services Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"David Lee","opportunity_name":"Lee Services Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/189","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_004","template_name":"convert_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/189"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/189"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/convert_lead"}],"source_schema_notes":[],"template_registry":{"name":"convert_lead","original":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nFor David Lee — complete the lead conversion (status should be 'Closed - Converted'). Account handling: if the lead’s company matches an existing Account, use it; if not, spin up a new Account using the company name from the lead. Then open an Opportunity named Lee Services Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_004","query_templated":"Convert the lead with name David Lee to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Lee Services Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nFor David Lee — complete the lead conversion (status should be 'Closed - Converted'). Account handling: if the lead’s company matches an existing Account, use it; if not, spin up a new Account using the company name from the lead. Then open an Opportunity named Lee Services Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"David Lee","opportunity_name":"Lee Services Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"David Lee","opportunity_name":"Lee Services Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/189"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L313","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_convert_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, IsConverted, ConvertedAccountId, ConvertedOpportunityId FROM Lead WHERE Name='{lead_name}'"}},{"type":"soql","id":"check_account","params":{"query":"SELECT Id, Name FROM Account WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedAccountId']}'"}},{"type":"soql","id":"opportunity_check","params":{"query":"SELECT Id, Name FROM Opportunity WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedOpportunityId']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9638706250f2e9ff6fc1593f5df7223033d03b97ec3ce5b8438d7053ba518783"} {"id":"scuba:sales_004_005","source_id":"scuba","original_id":"sales_004_005","kind":"benchmark_task","instruction":"Help me wrap up Grace King’s lead: convert it and set the status to 'Closed - Converted'. If there’s an Account with the same company name, associate the lead to that; otherwise create a new Account using the lead’s company name. Finally, create a new Opportunity called King Innovations Opportunity.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_005","query_templated":"Convert the lead with name Grace King to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name King Innovations Opportunity.","query":"Help me wrap up Grace King’s lead: convert it and set the status to 'Closed - Converted'. If there’s an Account with the same company name, associate the lead to that; otherwise create a new Account using the lead’s company name. Finally, create a new Opportunity called King Innovations Opportunity.","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Grace King","opportunity_name":"King Innovations Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Grace King","opportunity_name":"King Innovations Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/190","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_004","template_name":"convert_lead","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/190"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/190"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/convert_lead"}],"source_schema_notes":[],"template_registry":{"name":"convert_lead","original":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/convert_lead","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nHelp me wrap up Grace King’s lead: convert it and set the status to 'Closed - Converted'. If there’s an Account with the same company name, associate the lead to that; otherwise create a new Account using the lead’s company name. Finally, create a new Opportunity called King Innovations Opportunity.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_004_005","query_templated":"Convert the lead with name Grace King to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name King Innovations Opportunity.","query":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert.\n\n## Your ultimate task\n```\nHelp me wrap up Grace King’s lead: convert it and set the status to 'Closed - Converted'. If there’s an Account with the same company name, associate the lead to that; otherwise create a new Account using the lead’s company name. Finally, create a new Opportunity called King Innovations Opportunity.\n```","human_review_comment":"accept","ground_truth_dict":{"lead_name":"Grace King","opportunity_name":"King Innovations Opportunity"},"difficulty":"medium","query_template_name":"convert_lead","query_template_metadata":{"template_id":"sales_004","template_string":"Convert the lead with name {lead_name} to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name {opportunity_name}.","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Opportunity","Account","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Grace King","opportunity_name":"King Innovations Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Convert the lead with name Frank Knight to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Knight LLC Opportunity\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Leads”, and open the Leads app/list.\n- Open the lead “Frank Knight.”\n- Start conversion:\n - Set Lead Status to “Closed - Converted.”\n - Choose “Existing Account,” search for and select “Knight LLC.”\n - Enter Opportunity Name: “Knight LLC Opportunity.”\n- Click Convert to complete.\nSample 2:\nTASK:2. Convert the lead with name Jane Smith to 'Closed - Converted'. If the lead's company matches an existing account, select that account; otherwise create a new account with the lead's company name. Create a new opportunity with name Tech Giants Opportunity.\nEXPERIENCE:\nSuccessful plan to convert the lead and create an opportunity:\n\n- Log in to Salesforce.\n- Navigate to Leads > Recently Viewed and open the lead “Jane Smith.”\n- Start the lead conversion:\n - Open the Convert action/modal.\n - Set Lead Status to “Converted.”\n- Handle account selection:\n - Use Account Search and select the existing account “Tech Giants” (matching the lead’s company).\n- Create the opportunity:\n - In Opportunity Name, enter “Tech Giants Opportunity.”\n- Finalize by clicking Convert."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/190"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L313","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_convert_lead","data_retrieval_spec":[{"type":"soql","id":"lead_info","params":{"query":"SELECT Id, IsConverted, ConvertedAccountId, ConvertedOpportunityId FROM Lead WHERE Name='{lead_name}'"}},{"type":"soql","id":"check_account","params":{"query":"SELECT Id, Name FROM Account WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedAccountId']}'"}},{"type":"soql","id":"opportunity_check","params":{"query":"SELECT Id, Name FROM Opportunity WHERE Id='{retrieved_results['lead_info'].records[0]['ConvertedOpportunityId']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"1e82c7b5d2e276f2d93efd8ef566ee23c3f7ef16e3ed34c8d2a81223f8c7502f"} {"id":"scuba:sales_005_001","source_id":"scuba","original_id":"sales_005_001","kind":"benchmark_task","instruction":"For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline. Update the opportunity stage to Value Proposition.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_001","query_templated":"For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.","query":"For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline. Update the opportunity stage to Value Proposition.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"activity_type":"Event","opportunity_name":"Cloud Software Deal","activity_description":"Meeting with the client to discuss implementation timeline.","stage_name":"Value Proposition"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Cloud Software Deal","activity_description":"Schedule an event for meeting with the client to discuss implementation timeline.","stage_name":"Value Proposition"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/191","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_005","template_name":"update_opportunity_stage_and_activity","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/191"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/191"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/update_opportunity_stage_and_activity"}],"source_schema_notes":[],"template_registry":{"name":"update_opportunity_stage_and_activity","original":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline. Update the opportunity stage to Value Proposition.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_001","query_templated":"For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.","query":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline. Update the opportunity stage to Value Proposition.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"activity_type":"Event","opportunity_name":"Cloud Software Deal","activity_description":"Meeting with the client to discuss implementation timeline.","stage_name":"Value Proposition"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Cloud Software Deal","activity_description":"Schedule an event for meeting with the client to discuss implementation timeline.","stage_name":"Value Proposition"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/191"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L361","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_opportunity_stage_and_activity","data_retrieval_spec":[{"type":"soql","id":"opportunity_info","params":{"query":"SELECT Id, StageName FROM Opportunity WHERE Name='{opportunity_name}'"}},{"type":"soql","id":"task_info","params":{"query":"SELECT Id, Description, Subject, TaskSubtype FROM Task WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}},{"type":"soql","id":"event_info","params":{"query":"SELECT Id, Description, Subject, EventSubtype FROM Event WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5845405f5ec01b6d51956e748dfd3a291a1d8bf13dc0a764760fb356e521eae3"} {"id":"scuba:sales_005_002","source_id":"scuba","original_id":"sales_005_002","kind":"benchmark_task","instruction":"For Enterprise Software Sale, could you send a quick follow-up email to Confirm the proposal details, and afterward set the opportunity stage to Perception Analysis?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_002","query_templated":"For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.","query":"For Enterprise Software Sale, could you send a quick follow-up email to Confirm the proposal details, and afterward set the opportunity stage to Perception Analysis?","human_review_comment":"change to align with GT","ground_truth_dict":{"activity_type":"Email","opportunity_name":"Enterprise Software Sale","activity_description":"Email: Confirm the proposal details","stage_name":"Perception Analysis"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Enterprise Software Sale","activity_description":"Send a follow-up email to confirm the proposal details.","stage_name":"Perception Analysis"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/192","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_005","template_name":"update_opportunity_stage_and_activity","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/192"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/192"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/update_opportunity_stage_and_activity"}],"source_schema_notes":[],"template_registry":{"name":"update_opportunity_stage_and_activity","original":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor Enterprise Software Sale, could you send a quick follow-up email to Confirm the proposal details, and afterward set the opportunity stage to Perception Analysis?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_002","query_templated":"For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.","query":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor Enterprise Software Sale, could you send a quick follow-up email to Confirm the proposal details, and afterward set the opportunity stage to Perception Analysis?\n```","human_review_comment":"change to align with GT","ground_truth_dict":{"activity_type":"Email","opportunity_name":"Enterprise Software Sale","activity_description":"Email: Confirm the proposal details","stage_name":"Perception Analysis"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Enterprise Software Sale","activity_description":"Send a follow-up email to confirm the proposal details.","stage_name":"Perception Analysis"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/192"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L361","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_opportunity_stage_and_activity","data_retrieval_spec":[{"type":"soql","id":"opportunity_info","params":{"query":"SELECT Id, StageName FROM Opportunity WHERE Name='{opportunity_name}'"}},{"type":"soql","id":"task_info","params":{"query":"SELECT Id, Description, Subject, TaskSubtype FROM Task WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}},{"type":"soql","id":"event_info","params":{"query":"SELECT Id, Description, Subject, EventSubtype FROM Event WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5b28e49c9ed75ce1ac2c0c0b883c8edd92657c42c9ec7f66e81a373a89781d50"} {"id":"scuba:sales_005_003","source_id":"scuba","original_id":"sales_005_003","kind":"benchmark_task","instruction":"I just spoke with the product team to Review final requirements—log that call under the Product Development opportunity and move the stage to Negotiation/Review.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_003","query_templated":"For the opportunity Product Development, Log a call with the product team to review final requirements.. Update the opportunity stage to Negotiation/Review.","query":"I just spoke with the product team to Review final requirements—log that call under the Product Development opportunity and move the stage to Negotiation/Review.","human_review_comment":"change to align with GT","ground_truth_dict":{"activity_type":"Call","opportunity_name":"Product Development","activity_description":"Review final requirements","stage_name":"Negotiation/Review"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Product Development","activity_description":"Log a call with the product team to review final requirements.","stage_name":"Negotiation/Review"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/193","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_005","template_name":"update_opportunity_stage_and_activity","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/193"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/193"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/update_opportunity_stage_and_activity"}],"source_schema_notes":[],"template_registry":{"name":"update_opportunity_stage_and_activity","original":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI just spoke with the product team to Review final requirements—log that call under the Product Development opportunity and move the stage to Negotiation/Review.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_003","query_templated":"For the opportunity Product Development, Log a call with the product team to review final requirements.. Update the opportunity stage to Negotiation/Review.","query":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI just spoke with the product team to Review final requirements—log that call under the Product Development opportunity and move the stage to Negotiation/Review.\n```","human_review_comment":"change to align with GT","ground_truth_dict":{"activity_type":"Call","opportunity_name":"Product Development","activity_description":"Review final requirements","stage_name":"Negotiation/Review"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Product Development","activity_description":"Log a call with the product team to review final requirements.","stage_name":"Negotiation/Review"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/193"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L361","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_opportunity_stage_and_activity","data_retrieval_spec":[{"type":"soql","id":"opportunity_info","params":{"query":"SELECT Id, StageName FROM Opportunity WHERE Name='{opportunity_name}'"}},{"type":"soql","id":"task_info","params":{"query":"SELECT Id, Description, Subject, TaskSubtype FROM Task WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}},{"type":"soql","id":"event_info","params":{"query":"SELECT Id, Description, Subject, EventSubtype FROM Event WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"991b02e5c32db4f7e507db9936f88a49b4af9361ded60bb761cb27bf0faff039"} {"id":"scuba:sales_005_004","source_id":"scuba","original_id":"sales_005_004","kind":"benchmark_task","instruction":"First, under the Consulting Service opportunity, record this exact call summary: \"He wants to know the price for the product A & B.\" Next, change the stage to Proposal/Price Quote.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_004","query_templated":"For the opportunity Consulting Service, Log the call summary \"He wants to know the price for the product A & B.\". Update the opportunity stage to Proposal/Price Quote.","query":"First, under the Consulting Service opportunity, record this exact call summary: \"He wants to know the price for the product A & B.\" Next, change the stage to Proposal/Price Quote.","human_review_comment":"accept","ground_truth_dict":{"activity_type":"Call","opportunity_name":"Consulting Service","activity_description":"He wants to know the price for the product A & B.","stage_name":"Proposal/Price Quote"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Consulting Service","activity_description":"Log the call summary \"He wants to know the price for the product A & B.\"","stage_name":"Proposal/Price Quote"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/194","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_005","template_name":"update_opportunity_stage_and_activity","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/194"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/194"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/update_opportunity_stage_and_activity"}],"source_schema_notes":[],"template_registry":{"name":"update_opportunity_stage_and_activity","original":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFirst, under the Consulting Service opportunity, record this exact call summary: \"He wants to know the price for the product A & B.\" Next, change the stage to Proposal/Price Quote.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_004","query_templated":"For the opportunity Consulting Service, Log the call summary \"He wants to know the price for the product A & B.\". Update the opportunity stage to Proposal/Price Quote.","query":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFirst, under the Consulting Service opportunity, record this exact call summary: \"He wants to know the price for the product A & B.\" Next, change the stage to Proposal/Price Quote.\n```","human_review_comment":"accept","ground_truth_dict":{"activity_type":"Call","opportunity_name":"Consulting Service","activity_description":"He wants to know the price for the product A & B.","stage_name":"Proposal/Price Quote"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Consulting Service","activity_description":"Log the call summary \"He wants to know the price for the product A & B.\"","stage_name":"Proposal/Price Quote"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/194"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L361","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_opportunity_stage_and_activity","data_retrieval_spec":[{"type":"soql","id":"opportunity_info","params":{"query":"SELECT Id, StageName FROM Opportunity WHERE Name='{opportunity_name}'"}},{"type":"soql","id":"task_info","params":{"query":"SELECT Id, Description, Subject, TaskSubtype FROM Task WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}},{"type":"soql","id":"event_info","params":{"query":"SELECT Id, Description, Subject, EventSubtype FROM Event WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0d7e2a586c6cb970f7b932c8c57dc478c1d08536178888fbba9cd6e4b6e115f9"} {"id":"scuba:sales_005_005","source_id":"scuba","original_id":"sales_005_005","kind":"benchmark_task","instruction":"Create a new task on the Security Solutions opportunity with the subject line \"remind me search for the company's revenue,\" and update the stage to Needs Analysis.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_005","query_templated":"For the opportunity Security Solutions, Add a new task with the subject text \"remind me search for the company's revenue\". Update the opportunity stage to Needs Analysis.","query":"Create a new task on the Security Solutions opportunity with the subject line \"remind me search for the company's revenue,\" and update the stage to Needs Analysis.","human_review_comment":"accept","ground_truth_dict":{"activity_type":"Task","opportunity_name":"Security Solutions","activity_description":"remind me search for the company's revenue","stage_name":"Needs Analysis"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Security Solutions","activity_description":"Add a new task with the subject text \"remind me search for the company's revenue\"","stage_name":"Needs Analysis"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/195","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Leads & Opportunities","source_difficulty":"medium","template_id":"sales_005","template_name":"update_opportunity_stage_and_activity","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/195"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/195"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/update_opportunity_stage_and_activity"}],"source_schema_notes":[],"template_registry":{"name":"update_opportunity_stage_and_activity","original":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/update_opportunity_stage_and_activity","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate a new task on the Security Solutions opportunity with the subject line \"remind me search for the company's revenue,\" and update the stage to Needs Analysis.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Leads & Opportunities","task_id":"sales_005_005","query_templated":"For the opportunity Security Solutions, Add a new task with the subject text \"remind me search for the company's revenue\". Update the opportunity stage to Needs Analysis.","query":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate a new task on the Security Solutions opportunity with the subject line \"remind me search for the company's revenue,\" and update the stage to Needs Analysis.\n```","human_review_comment":"accept","ground_truth_dict":{"activity_type":"Task","opportunity_name":"Security Solutions","activity_description":"remind me search for the company's revenue","stage_name":"Needs Analysis"},"difficulty":"medium","query_template_name":"update_opportunity_stage_and_activity","query_template_metadata":{"template_id":"sales_005","template_string":"For the opportunity {opportunity_name}, {activity_description}. Update the opportunity stage to {stage_name}.","prerequisites":{"data":{"objects":["Opportunity"]}},"objects":["Opportunity","Task","Event"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Security Solutions","activity_description":"Add a new task with the subject text \"remind me search for the company's revenue\"","stage_name":"Needs Analysis"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:For the opportunity Cloud Software Deal, Schedule an event for meeting with the client to discuss implementation timeline.. Update the opportunity stage to Value Proposition.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Cloud Software Deal” opportunity.\n- Schedule the meeting event:\n - Click New Event.\n - Enter Subject: “Meeting to Discuss Implementation Timeline.”\n - Set Start Date: Aug 28, 2025.\n - Set End Date: Aug 28, 2025.\n - Click Save.\n- Update the opportunity stage:\n - Open Show more actions and click Edit.\n - Open the Opportunity Stage dropdown and select “Value Proposition.”\n - Click Save.\nSample 2:\nTASK:For the opportunity Enterprise Software Sale, Send a follow-up email to confirm the proposal details.. Update the opportunity stage to Perception Analysis.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM.\n- Navigate to Opportunities and open the “Enterprise Software Sale” opportunity.\n- Click Email to compose a message:\n - To: Jamie.taylor@company.com\n - Subject: Follow-Up: Confirmation of Proposal Details\n - Body: \n I hope this message finds you well.\n\n I wanted to follow up to confirm the details of the proposal we shared regarding the Enterprise Software Sale. Please let me know if you have any questions or need further clarification on any aspect.\n - Click Send.\n- Update the opportunity stage:\n - Select “Perception Analysis”.\n - Click “Mark as Current Stage”.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/195"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L361","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_update_opportunity_stage_and_activity","data_retrieval_spec":[{"type":"soql","id":"opportunity_info","params":{"query":"SELECT Id, StageName FROM Opportunity WHERE Name='{opportunity_name}'"}},{"type":"soql","id":"task_info","params":{"query":"SELECT Id, Description, Subject, TaskSubtype FROM Task WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}},{"type":"soql","id":"event_info","params":{"query":"SELECT Id, Description, Subject, EventSubtype FROM Event WHERE WhatId='{retrieved_results['opportunity_info'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4c157c4dd32872a7934886c4dac77f70ba7623535d9030d32e2aec4c1c5ddccf"} {"id":"scuba:sales_006_001","source_id":"scuba","original_id":"sales_006_001","kind":"benchmark_task","instruction":"Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_001","query_templated":"Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.","query":"Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Running Shoes","product_1_name":"ChillRun","product_1_description":"best fit for lightweight exercise","product_2_name":"MonsterRun","product_2_description":"designed for Marathon fans"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"Our company R&D department announced a new Product Family for \"Running Shoes\"","product_details":"For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/196","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_006","template_name":"create_product_family_and_products","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/196"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/196"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_product_family_and_products"}],"source_schema_notes":[],"template_registry":{"name":"create_product_family_and_products","original":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nOur company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_001","query_templated":"Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.","query":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nOur company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Running Shoes","product_1_name":"ChillRun","product_1_description":"best fit for lightweight exercise","product_2_name":"MonsterRun","product_2_description":"designed for Marathon fans"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"Our company R&D department announced a new Product Family for \"Running Shoes\"","product_details":"For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/196"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_product_family_and_products","data_retrieval_spec":[{"type":"soql","id":"product_family_check","params":{"query":"SELECT Id FROM Product2 WHERE Family='{product_family_name}'"}},{"type":"soql","id":"product1_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_1_name}'"}},{"type":"soql","id":"product2_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_2_name}'"}},{"type":"soql","id":"product3_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_3_name}'"}},{"type":"soql","id":"product4_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_4_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"77095551652f851c031b029243d9b221244d616740b2d8d5840c4f613d96e624"} {"id":"scuba:sales_006_002","source_id":"scuba","original_id":"sales_006_002","kind":"benchmark_task","instruction":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_002","query_templated":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.","query":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Smart Home Lighting Solutions","product_1_name":"BrightEase","product_1_description":"budget-friendly smart bulb for everyday use","product_2_name":"GlowSync","product_2_description":"features customizable color schemes and app control","product_3_name":"LuxEdge","product_3_description":"designed for premium interiors with motion sensing and voice assistant integration"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\"","product_details":"For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/197","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_006","template_name":"create_product_family_and_products","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/197"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/197"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_product_family_and_products"}],"source_schema_notes":[],"template_registry":{"name":"create_product_family_and_products","original":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nTechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_002","query_templated":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.","query":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nTechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Smart Home Lighting Solutions","product_1_name":"BrightEase","product_1_description":"budget-friendly smart bulb for everyday use","product_2_name":"GlowSync","product_2_description":"features customizable color schemes and app control","product_3_name":"LuxEdge","product_3_description":"designed for premium interiors with motion sensing and voice assistant integration"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\"","product_details":"For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"BrightEase\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/197"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_product_family_and_products","data_retrieval_spec":[{"type":"soql","id":"product_family_check","params":{"query":"SELECT Id FROM Product2 WHERE Family='{product_family_name}'"}},{"type":"soql","id":"product1_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_1_name}'"}},{"type":"soql","id":"product2_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_2_name}'"}},{"type":"soql","id":"product3_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_3_name}'"}},{"type":"soql","id":"product4_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_4_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"58d01db963d59846d128be0c9c4919e642d1b4660310927b72aa8fc8aa430d94"} {"id":"scuba:sales_006_003","source_id":"scuba","original_id":"sales_006_003","kind":"benchmark_task","instruction":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_003","query_templated":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.","query":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Electric Bike","product_1_name":"CityCruze","product_1_description":"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category","product_details":"For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/198","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_006","template_name":"create_product_family_and_products","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/198"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/198"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_product_family_and_products"}],"source_schema_notes":[],"template_registry":{"name":"create_product_family_and_products","original":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nEcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_003","query_templated":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.","query":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nEcoMotion has launched a new Product Family in the \"Electric Bike\" category. For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\". Help me to create the relevant info in the Product records.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Electric Bike","product_1_name":"CityCruze","product_1_description":"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"EcoMotion has launched a new Product Family in the \"Electric Bike\" category","product_details":"For this release, we have one product. The product \"CityCruze\" with the product code \"CityCruze\" is \"designed for daily commuters, offering pedal assist, lightweight frame, and a 40-mile battery range\""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/198"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_product_family_and_products","data_retrieval_spec":[{"type":"soql","id":"product_family_check","params":{"query":"SELECT Id FROM Product2 WHERE Family='{product_family_name}'"}},{"type":"soql","id":"product1_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_1_name}'"}},{"type":"soql","id":"product2_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_2_name}'"}},{"type":"soql","id":"product3_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_3_name}'"}},{"type":"soql","id":"product4_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_4_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6f60a9bc430c58b6267913a8482dd4d986b54fedb491c0615d366f7017158b10"} {"id":"scuba:sales_006_004","source_id":"scuba","original_id":"sales_006_004","kind":"benchmark_task","instruction":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_004","query_templated":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.","query":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Smart Home Security","product_1_name":"SmartLock Pro","product_1_description":"Advanced door security","product_2_name":"WatchCam 360","product_2_description":"Comprehensive surveillance","product_3_name":"AlertSense","product_3_description":"Motion detection and alerts"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.","product_details":"SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/199","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_006","template_name":"create_product_family_and_products","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/199"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/199"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_product_family_and_products"}],"source_schema_notes":[],"template_registry":{"name":"create_product_family_and_products","original":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nThe Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_004","query_templated":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.","query":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nThe Smart Home Security product family offers advanced solutions for home protection and monitoring.. SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts. Help me to create the relevant info in the Product records.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Smart Home Security","product_1_name":"SmartLock Pro","product_1_description":"Advanced door security","product_2_name":"WatchCam 360","product_2_description":"Comprehensive surveillance","product_3_name":"AlertSense","product_3_description":"Motion detection and alerts"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"The Smart Home Security product family offers advanced solutions for home protection and monitoring.","product_details":"SmartLock Pro for advanced door security, WatchCam 360 for comprehensive surveillance, and AlertSense for motion detection and alerts"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/199"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_product_family_and_products","data_retrieval_spec":[{"type":"soql","id":"product_family_check","params":{"query":"SELECT Id FROM Product2 WHERE Family='{product_family_name}'"}},{"type":"soql","id":"product1_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_1_name}'"}},{"type":"soql","id":"product2_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_2_name}'"}},{"type":"soql","id":"product3_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_3_name}'"}},{"type":"soql","id":"product4_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_4_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"364cadecba8de4073b5e0728fbfa9ac38bdb989f8fbae7dfdebb44ae7c2fbaf2"} {"id":"scuba:sales_006_005","source_id":"scuba","original_id":"sales_006_005","kind":"benchmark_task","instruction":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_005","query_templated":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.","query":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Professional Coffee Equipment","product_1_name":"BrewMaster Elite","product_1_description":"Commercial espresso brewing","product_2_name":"GrindTech Pro","product_2_description":"Precision bean grinding","product_3_name":"SteamForce","product_3_description":"Milk frothing","product_4_name":"CleanCycle","product_4_description":"Automated cleaning systems"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.","product_details":"BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/200","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_006","template_name":"create_product_family_and_products","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/200"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/200"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_product_family_and_products"}],"source_schema_notes":[],"template_registry":{"name":"create_product_family_and_products","original":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_product_family_and_products","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nThere is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_006_005","query_templated":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.","query":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save.\n\n## Your ultimate task\n```\nThere is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.. BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems. Help me to create the relevant info in the Product records.\n```","human_review_comment":"reject; use the orginal","ground_truth_dict":{"product_family_name":"Professional Coffee Equipment","product_1_name":"BrewMaster Elite","product_1_description":"Commercial espresso brewing","product_2_name":"GrindTech Pro","product_2_description":"Precision bean grinding","product_3_name":"SteamForce","product_3_description":"Milk frothing","product_4_name":"CleanCycle","product_4_description":"Automated cleaning systems"},"difficulty":"medium","query_template_name":"create_product_family_and_products","query_template_metadata":{"template_id":"sales_006","template_string":"{product_family_name_description}. {product_details}. Help me to create the relevant info in the Product records.","prerequisites":{},"objects":["Product2"],"metadata_types":["StandardValueSet"]},"instance_dict":{"product_family_name_description":"There is a new family of products focused on Professional Coffee Equipment, used for brewing, grinding, and maintaining coffee. It is designed for use in cafés and specialty coffee shops.","product_details":"BrewMaster Elite for commercial espresso brewing, GrindTech Pro for precision bean grinding, SteamForce for milk frothing, and CleanCycle for automated cleaning systems"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Our company R&D department announced a new Product Family for \"Running Shoes\". For this release, we have two products. The first product \"ChillRun\" with the product code \"CHILL-001\" is the \"best fit for lightweight exercise\"; and the second product \"MonsterRun\" with the product code \"MONS-001\" is \"designed for Marathon fans\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan:\n\n1) Add new Product Family value\n- Navigate: Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add picklist value: Running Shoes.\n- Save.\n\n2) Create product: ChillRun\n- Go to App Launcher > View All > Products.\n- Click New.\n- Set fields:\n - Product Name: ChillRun\n - Active: checked\n - Product Code: CHILL-001\n - Product Family: Running Shoes\n - Description: Best fit for lightweight exercise\n- Click Save & New.\n\n3) Create product: MonsterRun\n- In the new product form, set:\n - Product Name: MonsterRun\n - Active: checked\n - Product Code: MONS-001\n - Product Family: Running Shoes\n - Description: Designed for Marathon fans\n- Click Save.\nSample 2:\nTASK:TechNova has introduced a new Product Family focused on \"Smart Home Lighting Solutions\". For this release, we have three products. The first product \"BrightEase\" with the product code \"BrightEase\" is a \"budget-friendly smart bulb for everyday use\"; the second product \"GlowSync\" with the product code \"Glowsync\" \"features customizable color schemes and app control\"; and the third product \"LuxEdge\" with the product code \"LuxEdge\" is \"designed for premium interiors with motion sensing and voice assistant integration\". Help me to create the relevant info in the Product records.\nEXPERIENCE:\nSuccessful plan to create the Product Family and Products in Salesforce:\n\n1) Log in and add new Product Family picklist value\n- Go to Setup > Object Manager > Product > Fields & Relationships > Product Family.\n- Click New and add the picklist value: Smart Home Lighting Solutions.\n- Save.\n\n2) Create the three Product records\n- Open App Launcher, search for and open Products.\n\n- Product 1 (use Save & New):\n - New Product.\n - Product Name: BrightEase\n - Active: checked\n - Product Code: BrightEase\n - Product Family: Smart Home Lighting Solutions\n - Description: Budget-friendly smart bulb for everyday use.\n - Save & New.\n\n- Product 2 (use Save & New):\n - Product Name: GlowSync\n - Active: checked\n - Product Code: GlowSync\n - Product Family: Smart Home Lighting Solutions\n - Description: Features customizable color schemes and app control.\n - Save & New.\n\n- Product 3 (final Save):\n - Product Name: LuxEdge\n - Active: checked\n - Product Code: LuxEdge\n - Product Family: Smart Home Lighting Solutions\n - Description: Designed for premium interiors with motion sensing and voice assistant integration.\n - Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/200"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_product_family_and_products","data_retrieval_spec":[{"type":"soql","id":"product_family_check","params":{"query":"SELECT Id FROM Product2 WHERE Family='{product_family_name}'"}},{"type":"soql","id":"product1_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_1_name}'"}},{"type":"soql","id":"product2_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_2_name}'"}},{"type":"soql","id":"product3_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_3_name}'"}},{"type":"soql","id":"product4_check","params":{"query":"SELECT Id, Description, Family FROM Product2 WHERE Name='{product_4_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6963e1b2557dbdda17e1d7ba72f9a95f1008c9f8607c41b337d67eb76ef0dba9"} {"id":"scuba:sales_007_001","source_id":"scuba","original_id":"sales_007_001","kind":"benchmark_task","instruction":"Go ahead and set up two price books for the ThermalMax Pro—one named Holiday Sale at $150 per unit and another named Year End Special at $140 per unit. Make sure the specified product is included in each book with those exact unit prices.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_001","query_templated":"Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140.. Each price book should include the specified products with their respective unit prices.","query":"Go ahead and set up two price books for the ThermalMax Pro—one named Holiday Sale at $150 per unit and another named Year End Special at $140 per unit. Make sure the specified product is included in each book with those exact unit prices.","human_review_comment":"accept","ground_truth_dict":{"product_name":"ThermalMax Pro","price_book_name1":"Holiday Sale","unit_price1":150,"price_book_name2":"Year End Special","unit_price2":140},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/201","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_007","template_name":"create_price_books","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/201"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/201"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_price_books"}],"source_schema_notes":[],"template_registry":{"name":"create_price_books","original":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["PricebookEntry","Pricebook2"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books","differs_from_embedded_fields":["objects"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nGo ahead and set up two price books for the ThermalMax Pro—one named Holiday Sale at $150 per unit and another named Year End Special at $140 per unit. Make sure the specified product is included in each book with those exact unit prices.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_001","query_templated":"Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140.. Each price book should include the specified products with their respective unit prices.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nGo ahead and set up two price books for the ThermalMax Pro—one named Holiday Sale at $150 per unit and another named Year End Special at $140 per unit. Make sure the specified product is included in each book with those exact unit prices.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"ThermalMax Pro","price_book_name1":"Holiday Sale","unit_price1":150,"price_book_name2":"Year End Special","unit_price2":140},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/201"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L401","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_price_books","data_retrieval_spec":[{"type":"soql","id":"pricebook_info","params":{"query":"SELECT Name FROM Pricebook2 WHERE Name IN ('{price_book_name1}', '{price_book_name2}')"}},{"type":"soql","id":"pricebook_entry","params":{"query":"SELECT Product2.Name, Pricebook2.Name, UnitPrice FROM PricebookEntry WHERE Pricebook2.Name IN ('{price_book_name1}', '{price_book_name2}')"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"20b252dd69f64f52536d72ab85de8a1236c4033c8355ce908ea48cbb6d4af15d"} {"id":"scuba:sales_007_002","source_id":"scuba","original_id":"sales_007_002","kind":"benchmark_task","instruction":"Seasonal promos incoming—please create price books for CoolBreeze Elite: Summer Special priced at $120 per unit and Winter Deal priced at $110 per unit. Include the specified product in both price books, using those unit prices.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_002","query_templated":"Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110.. Each price book should include the specified products with their respective unit prices.","query":"Seasonal promos incoming—please create price books for CoolBreeze Elite: Summer Special priced at $120 per unit and Winter Deal priced at $110 per unit. Include the specified product in both price books, using those unit prices.","human_review_comment":"accept","ground_truth_dict":{"product_name":"CoolBreeze Elite","price_book_name1":"Summer Special","unit_price1":120,"price_book_name2":"Winter Deal","unit_price2":110},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/202","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_007","template_name":"create_price_books","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/202"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/202"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_price_books"}],"source_schema_notes":[],"template_registry":{"name":"create_price_books","original":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["PricebookEntry","Pricebook2"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books","differs_from_embedded_fields":["objects"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nSeasonal promos incoming—please create price books for CoolBreeze Elite: Summer Special priced at $120 per unit and Winter Deal priced at $110 per unit. Include the specified product in both price books, using those unit prices.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_002","query_templated":"Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110.. Each price book should include the specified products with their respective unit prices.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nSeasonal promos incoming—please create price books for CoolBreeze Elite: Summer Special priced at $120 per unit and Winter Deal priced at $110 per unit. Include the specified product in both price books, using those unit prices.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"CoolBreeze Elite","price_book_name1":"Summer Special","unit_price1":120,"price_book_name2":"Winter Deal","unit_price2":110},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/202"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L401","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_price_books","data_retrieval_spec":[{"type":"soql","id":"pricebook_info","params":{"query":"SELECT Name FROM Pricebook2 WHERE Name IN ('{price_book_name1}', '{price_book_name2}')"}},{"type":"soql","id":"pricebook_entry","params":{"query":"SELECT Product2.Name, Pricebook2.Name, UnitPrice FROM PricebookEntry WHERE Pricebook2.Name IN ('{price_book_name1}', '{price_book_name2}')"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a4f66496c8ee13a741e82d8db1e9694b027c885fa4406cdb9942904213abfb2d"} {"id":"scuba:sales_007_003","source_id":"scuba","original_id":"sales_007_003","kind":"benchmark_task","instruction":"Do two things for the Smart Fitness Tracker: 1) create a price book called Fitness Promo with a $75 unit price, and 2) create another called Wellness Special with a $70 unit price. Ensure the specified product is added to each price book at those unit prices.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_003","query_templated":"Create price books with the following details: Set up two price books for the product \"Smart Fitness Tracker\": one called \"Fitness Promo\" with a unit price of $75, and another called \"Wellness Special\" with a unit price of $70.. Each price book should include the specified products with their respective unit prices.","query":"Do two things for the Smart Fitness Tracker: 1) create a price book called Fitness Promo with a $75 unit price, and 2) create another called Wellness Special with a $70 unit price. Ensure the specified product is added to each price book at those unit prices.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Smart Fitness Tracker","price_book_name1":"Fitness Promo","unit_price1":75,"price_book_name2":"Wellness Special","unit_price2":70},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"Smart Fitness Tracker\": one called \"Fitness Promo\" with a unit price of $75, and another called \"Wellness Special\" with a unit price of $70."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/203","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_007","template_name":"create_price_books","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/203"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/203"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_price_books"}],"source_schema_notes":[],"template_registry":{"name":"create_price_books","original":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["PricebookEntry","Pricebook2"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books","differs_from_embedded_fields":["objects"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nDo two things for the Smart Fitness Tracker: 1) create a price book called Fitness Promo with a $75 unit price, and 2) create another called Wellness Special with a $70 unit price. Ensure the specified product is added to each price book at those unit prices.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_003","query_templated":"Create price books with the following details: Set up two price books for the product \"Smart Fitness Tracker\": one called \"Fitness Promo\" with a unit price of $75, and another called \"Wellness Special\" with a unit price of $70.. Each price book should include the specified products with their respective unit prices.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nDo two things for the Smart Fitness Tracker: 1) create a price book called Fitness Promo with a $75 unit price, and 2) create another called Wellness Special with a $70 unit price. Ensure the specified product is added to each price book at those unit prices.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Smart Fitness Tracker","price_book_name1":"Fitness Promo","unit_price1":75,"price_book_name2":"Wellness Special","unit_price2":70},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"Set up two price books for the product \"Smart Fitness Tracker\": one called \"Fitness Promo\" with a unit price of $75, and another called \"Wellness Special\" with a unit price of $70."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/203"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L401","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_price_books","data_retrieval_spec":[{"type":"soql","id":"pricebook_info","params":{"query":"SELECT Name FROM Pricebook2 WHERE Name IN ('{price_book_name1}', '{price_book_name2}')"}},{"type":"soql","id":"pricebook_entry","params":{"query":"SELECT Product2.Name, Pricebook2.Name, UnitPrice FROM PricebookEntry WHERE Pricebook2.Name IN ('{price_book_name1}', '{price_book_name2}')"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4b477f30e876a6c3a4246ad6484c48907c35af87c0bc87b262db0e4057422483"} {"id":"scuba:sales_007_004","source_id":"scuba","original_id":"sales_007_004","kind":"benchmark_task","instruction":"Set up member-tier price books: if it’s for silver members, price the Organic Skincare Set at $100 per unit; if it’s for golden members, set it to $80 per unit. Please include the specified product in each book with those unit prices.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_004","query_templated":"Create price books with the following details: I want to build custom price books for customers with different membership level. For silver members, the unit price for the product \"Organic Skincare Set\" is $100 and for the golden members, the unit price is $80.. Each price book should include the specified products with their respective unit prices.","query":"Set up member-tier price books: if it’s for silver members, price the Organic Skincare Set at $100 per unit; if it’s for golden members, set it to $80 per unit. Please include the specified product in each book with those unit prices.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Organic Skincare Set","price_book_name1":"Silver Members","unit_price1":100,"price_book_name2":"Golden Members","unit_price2":80},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"I want to build custom price books for customers with different membership level. For silver members, the unit price for the product \"Organic Skincare Set\" is $100 and for the golden members, the unit price is $80."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/204","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_007","template_name":"create_price_books","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/204"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/204"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_price_books"}],"source_schema_notes":[],"template_registry":{"name":"create_price_books","original":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["PricebookEntry","Pricebook2"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books","differs_from_embedded_fields":["objects"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nSet up member-tier price books: if it’s for silver members, price the Organic Skincare Set at $100 per unit; if it’s for golden members, set it to $80 per unit. Please include the specified product in each book with those unit prices.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_004","query_templated":"Create price books with the following details: I want to build custom price books for customers with different membership level. For silver members, the unit price for the product \"Organic Skincare Set\" is $100 and for the golden members, the unit price is $80.. Each price book should include the specified products with their respective unit prices.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nSet up member-tier price books: if it’s for silver members, price the Organic Skincare Set at $100 per unit; if it’s for golden members, set it to $80 per unit. Please include the specified product in each book with those unit prices.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Organic Skincare Set","price_book_name1":"Silver Members","unit_price1":100,"price_book_name2":"Golden Members","unit_price2":80},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"I want to build custom price books for customers with different membership level. For silver members, the unit price for the product \"Organic Skincare Set\" is $100 and for the golden members, the unit price is $80."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/204"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L401","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_price_books","data_retrieval_spec":[{"type":"soql","id":"pricebook_info","params":{"query":"SELECT Name FROM Pricebook2 WHERE Name IN ('{price_book_name1}', '{price_book_name2}')"}},{"type":"soql","id":"pricebook_entry","params":{"query":"SELECT Product2.Name, Pricebook2.Name, UnitPrice FROM PricebookEntry WHERE Pricebook2.Name IN ('{price_book_name1}', '{price_book_name2}')"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b99df9c16bc8262cfd0560a9d5682117e6e21354ee580b9ce3eabf77daba5f20"} {"id":"scuba:sales_007_005","source_id":"scuba","original_id":"sales_007_005","kind":"benchmark_task","instruction":"I’m implementing quantity-based pricing for the Titanium Pro Laptop—create two price books: Bulk Sale at $2,000 per unit and Starter Sale at $3,000 per unit. Include the specified product(s) in both price books at those unit prices.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_005","query_templated":"Create price books with the following details: I want to have different pricing strategy based on purchase quantity. Therefore, I need to create two Price books. One is named \"Bulk Sale\" and the unit price for the Titanium Pro Laptop is $2,000; and the other is named \"Starter Sale\" and the unit price should be $3,000.. Each price book should include the specified products with their respective unit prices.","query":"I’m implementing quantity-based pricing for the Titanium Pro Laptop—create two price books: Bulk Sale at $2,000 per unit and Starter Sale at $3,000 per unit. Include the specified product(s) in both price books at those unit prices.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Titanium Pro Laptop","price_book_name1":"Bulk Sale","unit_price1":2000,"price_book_name2":"Starter Sale","unit_price2":3000},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"I want to have different pricing strategy based on purchase quantity. Therefore, I need to create two Price books. One is named \"Bulk Sale\" and the unit price for the Titanium Pro Laptop is $2,000; and the other is named \"Starter Sale\" and the unit price should be $3,000."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/205","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"medium","template_id":"sales_007","template_name":"create_price_books","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/205"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/205"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_price_books"}],"source_schema_notes":[],"template_registry":{"name":"create_price_books","original":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["PricebookEntry","Pricebook2"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_price_books","differs_from_embedded_fields":["objects"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nI’m implementing quantity-based pricing for the Titanium Pro Laptop—create two price books: Bulk Sale at $2,000 per unit and Starter Sale at $3,000 per unit. Include the specified product(s) in both price books at those unit prices.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_007_005","query_templated":"Create price books with the following details: I want to have different pricing strategy based on purchase quantity. Therefore, I need to create two Price books. One is named \"Bulk Sale\" and the unit price for the Titanium Pro Laptop is $2,000; and the other is named \"Starter Sale\" and the unit price should be $3,000.. Each price book should include the specified products with their respective unit prices.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save.\n\n## Your ultimate task\n```\nI’m implementing quantity-based pricing for the Titanium Pro Laptop—create two price books: Bulk Sale at $2,000 per unit and Starter Sale at $3,000 per unit. Include the specified product(s) in both price books at those unit prices.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Titanium Pro Laptop","price_book_name1":"Bulk Sale","unit_price1":2000,"price_book_name2":"Starter Sale","unit_price2":3000},"difficulty":"medium","query_template_name":"create_price_books","query_template_metadata":{"template_id":"sales_007","template_string":"Create price books with the following details: {price_book_details}. Each price book should include the specified products with their respective unit prices.","prerequisites":{"data":{"objects":["Product2","PricebookEntry"]}},"objects":["Quote","Opportunity","PricebookEntry","Pricebook2"],"metadata_types":[]},"instance_dict":{"price_book_details":"I want to have different pricing strategy based on purchase quantity. Therefore, I need to create two Price books. One is named \"Bulk Sale\" and the unit price for the Titanium Pro Laptop is $2,000; and the other is named \"Starter Sale\" and the unit price should be $3,000."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create price books with the following details: Set up two price books for the product \"ThermalMax Pro\": one called \"Holiday Sale\" with a unit price of $150, and another called \"Year End Special\" with a unit price of $140. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open Price Books:\n - Click App Launcher > View All.\n - Search for “Price Books” and open it.\n- Create “Holiday Sale” price book:\n - Click New Price Book, set Name = “Holiday Sale”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 150, Save.\n- Create “Year End Special” price book:\n - Return to Price Books list, click New Price Book, set Name = “Year End Special”, check Active, Save.\n - Add product and price:\n - Go to Related tab > Add Products.\n - Select “ThermalMax Pro” > Next.\n - Clear the List Price field (press Backspace to remove the default value), enter 140, Save.\n\nNotes on hot-keys:\n- Used Backspace to clear the List Price field before entering the new unit prices.\nSample 2:\nTASK:Create price books with the following details: Set up two price books for the product \"CoolBreeze Elite\": one called \"Summer Special\" with a unit price of $120, and another called \"Winter Deal\" with a unit price of $110. Each price book should include the specified products with their respective unit prices.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Price Books:\n - App Launcher > View All > search “Price Books” > open.\n\n- Create “Summer Special” price book:\n - Click New, set Name = “Summer Special”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 120, Save.\n\n- Create “Winter Deal” price book:\n - From Price Books, click New, set Name = “Winter Deal”, mark Active, Save.\n - On the price book record: Related > Add Products.\n - Select product “CoolBreeze Elite” > Next.\n - In List Price field, clear existing value (used Backspace) and enter 110, Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/205"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L401","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_price_books","data_retrieval_spec":[{"type":"soql","id":"pricebook_info","params":{"query":"SELECT Name FROM Pricebook2 WHERE Name IN ('{price_book_name1}', '{price_book_name2}')"}},{"type":"soql","id":"pricebook_entry","params":{"query":"SELECT Product2.Name, Pricebook2.Name, UnitPrice FROM PricebookEntry WHERE Pricebook2.Name IN ('{price_book_name1}', '{price_book_name2}')"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f8bca210c898b6e1ee3d881bdd8ff6c9c0f9ecb5b7221d11e0f78867985ada11"} {"id":"scuba:sales_008_001","source_id":"scuba","original_id":"sales_008_001","kind":"benchmark_task","instruction":"The prospect is really into ThermalMax Pro—please spin up a quote titled \"Holiday Sale\" on the \"Product Development\" opportunity. Set it to expire 10 days from today. Make the final sales price equal to list price minus 10%, and only honor that pricing when the quantity is 10.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_001","query_templated":"The client shows a great interest in the product 'ThermalMax Pro'. Create a quote with the name 'Holiday Sale' for the opportunity 'Product Development' which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.","query":"The prospect is really into ThermalMax Pro—please spin up a quote titled \"Holiday Sale\" on the \"Product Development\" opportunity. Set it to expire 10 days from today. Make the final sales price equal to list price minus 10%, and only honor that pricing when the quantity is 10.","human_review_comment":"accept","ground_truth_dict":{"product_name":"ThermalMax Pro","quote_name":"Holiday Sale","opportunity_name":"Product Development","expiration_duration":"tendays","discount_percentage":10,"quantity":10},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"ThermalMax Pro","quote_name":"Holiday Sale","opportunity_name":"Product Development","expiration_duration":"10 days","discount_percentage":10,"quantity":10},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/206","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_008","template_name":"create_quote","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/206"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/206"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_quote"}],"source_schema_notes":[],"template_registry":{"name":"create_quote","original":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe prospect is really into ThermalMax Pro—please spin up a quote titled \"Holiday Sale\" on the \"Product Development\" opportunity. Set it to expire 10 days from today. Make the final sales price equal to list price minus 10%, and only honor that pricing when the quantity is 10.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_001","query_templated":"The client shows a great interest in the product 'ThermalMax Pro'. Create a quote with the name 'Holiday Sale' for the opportunity 'Product Development' which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.","query":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe prospect is really into ThermalMax Pro—please spin up a quote titled \"Holiday Sale\" on the \"Product Development\" opportunity. Set it to expire 10 days from today. Make the final sales price equal to list price minus 10%, and only honor that pricing when the quantity is 10.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"ThermalMax Pro","quote_name":"Holiday Sale","opportunity_name":"Product Development","expiration_duration":"tendays","discount_percentage":10,"quantity":10},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"ThermalMax Pro","quote_name":"Holiday Sale","opportunity_name":"Product Development","expiration_duration":"10 days","discount_percentage":10,"quantity":10},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/206"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L114","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_quote","data_retrieval_spec":[{"type":"soql","id":"quote_info","params":{"query":"SELECT Id, ExpirationDate, Opportunity.Name FROM Quote WHERE Name='{quote_name}'"}},{"type":"soql","id":"quote_line_item","params":{"query":"SELECT Discount, Product2.Name, Quantity FROM QuoteLineItem WHERE Quote.Name='{quote_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"978d3d2c02c5436f76290cd5115edd2317c58d9a0b48ab6b8fb90915495465db"} {"id":"scuba:sales_008_002","source_id":"scuba","original_id":"sales_008_002","kind":"benchmark_task","instruction":"The customer’s excited about CoolBreeze Elite. Could you create a \"Summer Special\" quote tied to the \"Maintenance Contract\" opportunity that expires one week from today? Set the final sales price by taking 3.5% off the list price, with the discount applicable only if they order 14 units.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_002","query_templated":"The client shows a great interest in the product 'CoolBreeze Elite'. Create a quote with the name 'Summer Special' for the opportunity 'Maintenance Contract' which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.","query":"The customer’s excited about CoolBreeze Elite. Could you create a \"Summer Special\" quote tied to the \"Maintenance Contract\" opportunity that expires one week from today? Set the final sales price by taking 3.5% off the list price, with the discount applicable only if they order 14 units.","human_review_comment":"accept","ground_truth_dict":{"product_name":"CoolBreeze Elite","quote_name":"Summer Special","opportunity_name":"Maintenance Contract","expiration_duration":"oneweek","discount_percentage":3.5,"quantity":14},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"CoolBreeze Elite","quote_name":"Summer Special","opportunity_name":"Maintenance Contract","expiration_duration":"1 week","discount_percentage":3.5,"quantity":14},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/207","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_008","template_name":"create_quote","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/207"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/207"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_quote"}],"source_schema_notes":[],"template_registry":{"name":"create_quote","original":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe customer’s excited about CoolBreeze Elite. Could you create a \"Summer Special\" quote tied to the \"Maintenance Contract\" opportunity that expires one week from today? Set the final sales price by taking 3.5% off the list price, with the discount applicable only if they order 14 units.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_002","query_templated":"The client shows a great interest in the product 'CoolBreeze Elite'. Create a quote with the name 'Summer Special' for the opportunity 'Maintenance Contract' which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.","query":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe customer’s excited about CoolBreeze Elite. Could you create a \"Summer Special\" quote tied to the \"Maintenance Contract\" opportunity that expires one week from today? Set the final sales price by taking 3.5% off the list price, with the discount applicable only if they order 14 units.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"CoolBreeze Elite","quote_name":"Summer Special","opportunity_name":"Maintenance Contract","expiration_duration":"oneweek","discount_percentage":3.5,"quantity":14},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"CoolBreeze Elite","quote_name":"Summer Special","opportunity_name":"Maintenance Contract","expiration_duration":"1 week","discount_percentage":3.5,"quantity":14},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/207"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L114","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_quote","data_retrieval_spec":[{"type":"soql","id":"quote_info","params":{"query":"SELECT Id, ExpirationDate, Opportunity.Name FROM Quote WHERE Name='{quote_name}'"}},{"type":"soql","id":"quote_line_item","params":{"query":"SELECT Discount, Product2.Name, Quantity FROM QuoteLineItem WHERE Quote.Name='{quote_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"357743c69c204a4aa09d68c3c82bb4b411f1447db932ef5aa610dc01012cb2ab"} {"id":"scuba:sales_008_003","source_id":"scuba","original_id":"sales_008_003","kind":"benchmark_task","instruction":"Lots of buzz around the Smart Fitness Tracker. Draft a quote named \"Fitness Promo\" for the \"IT Outsourcing\" opportunity; have it expire 5 days from today. Final sales price should be list less 13.2%, and this rate should only qualify at a quantity of 35.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_003","query_templated":"The client shows a great interest in the product 'Smart Fitness Tracker'. Create a quote with the name 'Fitness Promo' for the opportunity 'IT Outsourcing' which expires 5 days from today. Let's apply 13.2% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 35.","query":"Lots of buzz around the Smart Fitness Tracker. Draft a quote named \"Fitness Promo\" for the \"IT Outsourcing\" opportunity; have it expire 5 days from today. Final sales price should be list less 13.2%, and this rate should only qualify at a quantity of 35.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Smart Fitness Tracker","quote_name":"Fitness Promo","opportunity_name":"IT Outsourcing","expiration_duration":"fivedays","discount_percentage":13.2,"quantity":35},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Smart Fitness Tracker","quote_name":"Fitness Promo","opportunity_name":"IT Outsourcing","expiration_duration":"5 days","discount_percentage":13.2,"quantity":35},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/208","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_008","template_name":"create_quote","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/208"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/208"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_quote"}],"source_schema_notes":[],"template_registry":{"name":"create_quote","original":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nLots of buzz around the Smart Fitness Tracker. Draft a quote named \"Fitness Promo\" for the \"IT Outsourcing\" opportunity; have it expire 5 days from today. Final sales price should be list less 13.2%, and this rate should only qualify at a quantity of 35.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_003","query_templated":"The client shows a great interest in the product 'Smart Fitness Tracker'. Create a quote with the name 'Fitness Promo' for the opportunity 'IT Outsourcing' which expires 5 days from today. Let's apply 13.2% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 35.","query":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nLots of buzz around the Smart Fitness Tracker. Draft a quote named \"Fitness Promo\" for the \"IT Outsourcing\" opportunity; have it expire 5 days from today. Final sales price should be list less 13.2%, and this rate should only qualify at a quantity of 35.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Smart Fitness Tracker","quote_name":"Fitness Promo","opportunity_name":"IT Outsourcing","expiration_duration":"fivedays","discount_percentage":13.2,"quantity":35},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Smart Fitness Tracker","quote_name":"Fitness Promo","opportunity_name":"IT Outsourcing","expiration_duration":"5 days","discount_percentage":13.2,"quantity":35},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/208"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L114","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_quote","data_retrieval_spec":[{"type":"soql","id":"quote_info","params":{"query":"SELECT Id, ExpirationDate, Opportunity.Name FROM Quote WHERE Name='{quote_name}'"}},{"type":"soql","id":"quote_line_item","params":{"query":"SELECT Discount, Product2.Name, Quantity FROM QuoteLineItem WHERE Quote.Name='{quote_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"cd508c861eb679acc5408e9c034b74c6b925ef0dde4173b80f6281d7c152625d"} {"id":"scuba:sales_008_004","source_id":"scuba","original_id":"sales_008_004","kind":"benchmark_task","instruction":"We’ve got strong interest in the Organic Skincare Set. Please issue a quote called \"Silver Members\" linked to the \"Digital Marketing\" opportunity, expiring 2 weeks from today. Set the final sales price by applying a 15.4% discount to the list price, and require a purchase quantity of 20 to qualify.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_004","query_templated":"The client shows a great interest in the product 'Organic Skincare Set'. Create a quote with the name 'Silver Members' for the opportunity 'Digital Marketing' which expires 2 weeks from today. Let's apply 15.4% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 20.","query":"We’ve got strong interest in the Organic Skincare Set. Please issue a quote called \"Silver Members\" linked to the \"Digital Marketing\" opportunity, expiring 2 weeks from today. Set the final sales price by applying a 15.4% discount to the list price, and require a purchase quantity of 20 to qualify.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Organic Skincare Set","quote_name":"Silver Members","opportunity_name":"Digital Marketing","expiration_duration":"twoweeks","discount_percentage":15.4,"quantity":20},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Organic Skincare Set","quote_name":"Silver Members","opportunity_name":"Digital Marketing","expiration_duration":"2 weeks","discount_percentage":15.4,"quantity":20},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/209","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_008","template_name":"create_quote","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/209"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/209"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_quote"}],"source_schema_notes":[],"template_registry":{"name":"create_quote","original":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nWe’ve got strong interest in the Organic Skincare Set. Please issue a quote called \"Silver Members\" linked to the \"Digital Marketing\" opportunity, expiring 2 weeks from today. Set the final sales price by applying a 15.4% discount to the list price, and require a purchase quantity of 20 to qualify.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_004","query_templated":"The client shows a great interest in the product 'Organic Skincare Set'. Create a quote with the name 'Silver Members' for the opportunity 'Digital Marketing' which expires 2 weeks from today. Let's apply 15.4% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 20.","query":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nWe’ve got strong interest in the Organic Skincare Set. Please issue a quote called \"Silver Members\" linked to the \"Digital Marketing\" opportunity, expiring 2 weeks from today. Set the final sales price by applying a 15.4% discount to the list price, and require a purchase quantity of 20 to qualify.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Organic Skincare Set","quote_name":"Silver Members","opportunity_name":"Digital Marketing","expiration_duration":"twoweeks","discount_percentage":15.4,"quantity":20},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Organic Skincare Set","quote_name":"Silver Members","opportunity_name":"Digital Marketing","expiration_duration":"2 weeks","discount_percentage":15.4,"quantity":20},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/209"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L114","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_quote","data_retrieval_spec":[{"type":"soql","id":"quote_info","params":{"query":"SELECT Id, ExpirationDate, Opportunity.Name FROM Quote WHERE Name='{quote_name}'"}},{"type":"soql","id":"quote_line_item","params":{"query":"SELECT Discount, Product2.Name, Quantity FROM QuoteLineItem WHERE Quote.Name='{quote_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4cfe2ff2a16f109ec4828733d7e01b62767184bed4b00c77c001440192352874"} {"id":"scuba:sales_008_005","source_id":"scuba","original_id":"sales_008_005","kind":"benchmark_task","instruction":"The client is keen on the Titanium Pro Laptop. Prepare a \"Bulk Sale\" quote for the \"Supply Chain Project\" opportunity that’s valid for 1 month from today. Final sales price = list price minus 15%, and make this pricing contingent on buying 2 units.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_005","query_templated":"The client shows a great interest in the product 'Titanium Pro Laptop'. Create a quote with the name 'Bulk Sale' for the opportunity 'Supply Chain Project' which expires 1 month from today. Let's apply 15% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 2.","query":"The client is keen on the Titanium Pro Laptop. Prepare a \"Bulk Sale\" quote for the \"Supply Chain Project\" opportunity that’s valid for 1 month from today. Final sales price = list price minus 15%, and make this pricing contingent on buying 2 units.","human_review_comment":"accept","ground_truth_dict":{"product_name":"Titanium Pro Laptop","quote_name":"Bulk Sale","opportunity_name":"Supply Chain Project","expiration_duration":"onemonth","discount_percentage":15,"quantity":2},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Titanium Pro Laptop","quote_name":"Bulk Sale","opportunity_name":"Supply Chain Project","expiration_duration":"1 month","discount_percentage":15,"quantity":2},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/210","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_008","template_name":"create_quote","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/210"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/210"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_quote"}],"source_schema_notes":[],"template_registry":{"name":"create_quote","original":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_quote","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe client is keen on the Titanium Pro Laptop. Prepare a \"Bulk Sale\" quote for the \"Supply Chain Project\" opportunity that’s valid for 1 month from today. Final sales price = list price minus 15%, and make this pricing contingent on buying 2 units.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_008_005","query_templated":"The client shows a great interest in the product 'Titanium Pro Laptop'. Create a quote with the name 'Bulk Sale' for the opportunity 'Supply Chain Project' which expires 1 month from today. Let's apply 15% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 2.","query":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote.\n\n## Your ultimate task\n```\nThe client is keen on the Titanium Pro Laptop. Prepare a \"Bulk Sale\" quote for the \"Supply Chain Project\" opportunity that’s valid for 1 month from today. Final sales price = list price minus 15%, and make this pricing contingent on buying 2 units.\n```","human_review_comment":"accept","ground_truth_dict":{"product_name":"Titanium Pro Laptop","quote_name":"Bulk Sale","opportunity_name":"Supply Chain Project","expiration_duration":"onemonth","discount_percentage":15,"quantity":2},"difficulty":"hard","query_template_name":"create_quote","query_template_metadata":{"template_id":"sales_008","template_string":"The client shows a great interest in the product '{product_name}'. Create a quote with the name '{quote_name}' for the opportunity '{opportunity_name}' which expires {expiration_duration} from today. Let's apply {discount_percentage}% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be {quantity}.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Quote"]}},"data":{"objects":["Product2","PricebookEntry","Opportunity"]}},"objects":["Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"product_name":"Titanium Pro Laptop","quote_name":"Bulk Sale","opportunity_name":"Supply Chain Project","expiration_duration":"1 month","discount_percentage":15,"quantity":2},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The client shows a great interest in the product ThermalMax Pro. Create a quote with the name Holiday Sale for the opportunity Product Development which expires 10 days from today. Let's apply 10% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 10.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “Opportunities,” and open it.\n- Open the “Product Development” opportunity.\n- In the Quotes related list, click “New Quote.”\n- Set:\n - Quote Name: “Holiday Sale”\n - Expiration Date: Today + 10 days\n - Save the quote.\n- Open the newly created “Holiday Sale” quote.\n- Click “Add Products.”\n - If prompted, confirm/save the price book.\n - Select the product “ThermalMax Pro” and click “Next.”\n- In the line editor:\n - Set Quantity to 10.\n - Set Discount (Percentage) to 10.\n - Save.\n\nNotes:\n- No hot-keys were used.\n- The 10% discount is applied to the list price to determine the final sales price.\nSample 2:\nTASK:The client shows a great interest in the product CoolBreeze Elite. Create a quote with the name Summer Special for the opportunity Maintenance Contract which expires 1 week from today. Let's apply 3.5% discount on the list price to make it the final Sales price. To qualify this price, the quantity needs to be 14.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the CRM and open Opportunities.\n- Open the “Maintenance Contract” opportunity.\n- In the Quotes related list, click New Quote.\n- Set Quote Name to “Summer Special” and set Expiration Date to one week from today; Save.\n- Open the newly created “Summer Special” quote.\n- Click Add Products, select “CoolBreeze Elite,” then click Next.\n- Edit the quote line:\n - Set Quantity to 14.\n - Set Discount (Percentage) to 3.5 to derive the final Sales price from the list price.\n- Save the quote."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/210"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L114","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_quote","data_retrieval_spec":[{"type":"soql","id":"quote_info","params":{"query":"SELECT Id, ExpirationDate, Opportunity.Name FROM Quote WHERE Name='{quote_name}'"}},{"type":"soql","id":"quote_line_item","params":{"query":"SELECT Discount, Product2.Name, Quantity FROM QuoteLineItem WHERE Quote.Name='{quote_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9fbe408fa74e32f893ec9149d8382abbeaa0f58de2bbb110915422efe205bd9b"} {"id":"scuba:sales_009_001","source_id":"scuba","original_id":"sales_009_001","kind":"benchmark_task","instruction":"Could you take care of the Knight LLC account for me? Build a 12‑month contract that begins today. After that’s created, open an order from this contract and set its start date to be the next Friday after the contract’s start date.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_001","query_templated":"Prepare a contract for the account Knight LLC. The start date is today. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.","query":"Could you take care of the Knight LLC account for me? Build a 12‑month contract that begins today. After that’s created, open an order from this contract and set its start date to be the next Friday after the contract’s start date.","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Knight LLC","start_date":"today","contract_term_months":12,"order_start_date":"next friday"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Knight LLC","start_date":"today","contract_term_months":12,"order_start_date":"the Friday following the contract start date"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/211","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_009","template_name":"create_contract_and_order","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/211"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/211"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_contract_and_order"}],"source_schema_notes":[],"template_registry":{"name":"create_contract_and_order","original":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you take care of the Knight LLC account for me? Build a 12‑month contract that begins today. After that’s created, open an order from this contract and set its start date to be the next Friday after the contract’s start date.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_001","query_templated":"Prepare a contract for the account Knight LLC. The start date is today. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.","query":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you take care of the Knight LLC account for me? Build a 12‑month contract that begins today. After that’s created, open an order from this contract and set its start date to be the next Friday after the contract’s start date.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Knight LLC","start_date":"today","contract_term_months":12,"order_start_date":"next friday"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Knight LLC","start_date":"today","contract_term_months":12,"order_start_date":"the Friday following the contract start date"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/211"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L454","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_contract_and_order","data_retrieval_spec":[{"type":"soql","id":"contract_creation","params":{"query":"SELECT Id, StartDate, ContractTerm FROM Contract WHERE Account.Name='{account_name}'"}},{"type":"soql","id":"order_creation","params":{"query":"SELECT Id, EffectiveDate FROM Order WHERE ContractId='{retrieved_results['contract_creation'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9ce8a6d48a72f82f33faaaeb900f468859246802c69c370339b2e5c5bf683247"} {"id":"scuba:sales_009_002","source_id":"scuba","original_id":"sales_009_002","kind":"benchmark_task","instruction":"On the Lee Services account, set up a contract with a 24‑month term starting on the next week. Once it’s in place, generate an order tied to that contract and have the order go live on three weeks after contract date.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_002","query_templated":"Prepare a contract for the account Lee Services. The start date is next week. The contract term is 24 months. Then create an order based on this contract. The order start date should be three weeks after contract date.","query":"On the Lee Services account, set up a contract with a 24‑month term starting on the next week. Once it’s in place, generate an order tied to that contract and have the order go live on three weeks after contract date.","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Lee Services","start_date":"1 week","contract_term_months":24,"order_start_date":"3 weeks"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":"{'account_name': 'Lee Services', 'start_date': 'next week': '24', 'order_start_date': 'three weeks after contract start date'}","on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/212","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_009","template_name":"create_contract_and_order","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/212"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/212"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_contract_and_order"}],"source_schema_notes":["Known source instance_dict is a malformed Python-like string, preserved verbatim without parsing or repair; grading uses ground_truth_dict."],"template_registry":{"name":"create_contract_and_order","original":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nOn the Lee Services account, set up a contract with a 24‑month term starting on the next week. Once it’s in place, generate an order tied to that contract and have the order go live on three weeks after contract date.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_002","query_templated":"Prepare a contract for the account Lee Services. The start date is next week. The contract term is 24 months. Then create an order based on this contract. The order start date should be three weeks after contract date.","query":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nOn the Lee Services account, set up a contract with a 24‑month term starting on the next week. Once it’s in place, generate an order tied to that contract and have the order go live on three weeks after contract date.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Lee Services","start_date":"1 week","contract_term_months":24,"order_start_date":"3 weeks"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":"{'account_name': 'Lee Services', 'start_date': 'next week': '24', 'order_start_date': 'three weeks after contract start date'}","on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/212"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L454","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_contract_and_order","data_retrieval_spec":[{"type":"soql","id":"contract_creation","params":{"query":"SELECT Id, StartDate, ContractTerm FROM Contract WHERE Account.Name='{account_name}'"}},{"type":"soql","id":"order_creation","params":{"query":"SELECT Id, EffectiveDate FROM Order WHERE ContractId='{retrieved_results['contract_creation'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f06b2c3ce3b98382dcd6181d9d22a84046b03cbbf9c1ffdede17f6726633f0fa"} {"id":"scuba:sales_009_003","source_id":"scuba","original_id":"sales_009_003","kind":"benchmark_task","instruction":"For the Green Solutions account, please: 1) Create a contract that starts in 1 month from today. 2) Make the term 18 months. 3) From that contract, create an associated order that begins on 1 month after the contract start date.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_003","query_templated":"Prepare a contract for the account Green Solutions. The start date is next month. The contract term is 18 months. Then create an order based on this contract. The order start date should be a month after contract start date.","query":"For the Green Solutions account, please: 1) Create a contract that starts in 1 month from today. 2) Make the term 18 months. 3) From that contract, create an associated order that begins on 1 month after the contract start date.","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Green Solutions","start_date":"1 month","contract_term_months":18,"order_start_date":"1 month"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Green Solutions","start_date":"next month","contract_term_months":18,"order_start_date":" a month after contract start date"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/213","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_009","template_name":"create_contract_and_order","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/213"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/213"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_contract_and_order"}],"source_schema_notes":[],"template_registry":{"name":"create_contract_and_order","original":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the Green Solutions account, please: 1) Create a contract that starts in 1 month from today. 2) Make the term 18 months. 3) From that contract, create an associated order that begins on 1 month after the contract start date.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_003","query_templated":"Prepare a contract for the account Green Solutions. The start date is next month. The contract term is 18 months. Then create an order based on this contract. The order start date should be a month after contract start date.","query":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nFor the Green Solutions account, please: 1) Create a contract that starts in 1 month from today. 2) Make the term 18 months. 3) From that contract, create an associated order that begins on 1 month after the contract start date.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Green Solutions","start_date":"1 month","contract_term_months":18,"order_start_date":"1 month"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Green Solutions","start_date":"next month","contract_term_months":18,"order_start_date":" a month after contract start date"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/213"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L454","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_contract_and_order","data_retrieval_spec":[{"type":"soql","id":"contract_creation","params":{"query":"SELECT Id, StartDate, ContractTerm FROM Contract WHERE Account.Name='{account_name}'"}},{"type":"soql","id":"order_creation","params":{"query":"SELECT Id, EffectiveDate FROM Order WHERE ContractId='{retrieved_results['contract_creation'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a9f20e7ef6d71c918ed7562cb2e50d2aa111fb4e265ea01396b4bb73359fe17b"} {"id":"scuba:sales_009_004","source_id":"scuba","original_id":"sales_009_004","kind":"benchmark_task","instruction":"Prepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_004","query_templated":"Prepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.","query":"Prepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Tech Giants","start_date":"3 weeks","contract_term_months":12,"order_start_date":"1 day"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Tech Giants","start_date":"three weeks from today","contract_term_months":12,"order_start_date":"one day after the Contract start date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/214","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_009","template_name":"create_contract_and_order","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/214"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/214"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_contract_and_order"}],"source_schema_notes":[],"template_registry":{"name":"create_contract_and_order","original":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPrepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_004","query_templated":"Prepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.","query":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPrepare a contract for the account Tech Giants. The start date is three weeks from today. The contract term is 12 months. Then create an order based on this contract. The order start date should be one day after the Contract start date.\n```","human_review_comment":"modified multiple palces","ground_truth_dict":{"account_name":"Tech Giants","start_date":"3 weeks","contract_term_months":12,"order_start_date":"1 day"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Tech Giants","start_date":"three weeks from today","contract_term_months":12,"order_start_date":"one day after the Contract start date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/214"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L454","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_contract_and_order","data_retrieval_spec":[{"type":"soql","id":"contract_creation","params":{"query":"SELECT Id, StartDate, ContractTerm FROM Contract WHERE Account.Name='{account_name}'"}},{"type":"soql","id":"order_creation","params":{"query":"SELECT Id, EffectiveDate FROM Order WHERE ContractId='{retrieved_results['contract_creation'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a54a8e1eb7d960a5eb6c97a358e271a68f1390958834dd3f5d59bada42b71d6b"} {"id":"scuba:sales_009_005","source_id":"scuba","original_id":"sales_009_005","kind":"benchmark_task","instruction":"Let’s handle the Black Enterprises account like this: set up a contract starting in 4 weeks with a 6‑month term. Then create an order from that contract and schedule it to begin two weeks after the contract start date.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_005","query_templated":"Prepare a contract for the account Black Enterprises. The start date is in four weeks. The contract term is 6 months. Then create an order based on this contract. The order start date should be two weeks after the Contract start date.","query":"Let’s handle the Black Enterprises account like this: set up a contract starting in 4 weeks with a 6‑month term. Then create an order from that contract and schedule it to begin two weeks after the contract start date.","human_review_comment":null,"ground_truth_dict":{"account_name":"Black Enterprises","start_date":"4 weeks","contract_term_months":6,"order_start_date":"2 weeks"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Black Enterprises","start_date":"in four weeks","contract_term_months":6,"order_start_date":"two weeks after the Contract start date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/215","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"CPQ","source_difficulty":"hard","template_id":"sales_009","template_name":"create_contract_and_order","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/215"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/215"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_contract_and_order"}],"source_schema_notes":[],"template_registry":{"name":"create_contract_and_order","original":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_contract_and_order","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s handle the Black Enterprises account like this: set up a contract starting in 4 weeks with a 6‑month term. Then create an order from that contract and schedule it to begin two weeks after the contract start date.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"CPQ","task_id":"sales_009_005","query_templated":"Prepare a contract for the account Black Enterprises. The start date is in four weeks. The contract term is 6 months. Then create an order based on this contract. The order start date should be two weeks after the Contract start date.","query":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s handle the Black Enterprises account like this: set up a contract starting in 4 weeks with a 6‑month term. Then create an order from that contract and schedule it to begin two weeks after the contract start date.\n```","human_review_comment":null,"ground_truth_dict":{"account_name":"Black Enterprises","start_date":"4 weeks","contract_term_months":6,"order_start_date":"2 weeks"},"difficulty":"hard","query_template_name":"create_contract_and_order","query_template_metadata":{"template_id":"sales_009","template_string":"Prepare a contract for the account {account_name}. The start date is {start_date}. The contract term is {contract_term_months} months. Then create an order based on this contract. The order start date should be {order_start_date}.","prerequisites":{"data":{"objects":["Account"]}},"objects":["Order","Contract"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Black Enterprises","start_date":"in four weeks","contract_term_months":6,"order_start_date":"two weeks after the Contract start date"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Prepare a contract for the account Knight LLC. The start date is the third Friday of August, 2025. The contract term is 12 months. Then create an order based on this contract. The order start date should be the Friday following the contract start date.\nEXPERIENCE:\nSuccessful plan:\n\n1) Access Contracts\n- Log in.\n- Open App Launcher > View All > search “contracts” > open Contracts.\n\n2) Create the contract for Knight LLC\n- Click New.\n- In the Account lookup, select Knight LLC.\n- Set Start Date to 8/15/2025 (third Friday of August 2025).\n- Set Contract Term to 12 months.\n- Save.\n\n3) Create the order based on the contract\n- Open App Launcher > View All > search “orders” > open Orders.\n- Click New Order.\n- In the Account lookup, select Knight LLC.\n- Set Order Start Date to 8/22/2025 (the Friday following the contract start date).\n- In the Contract lookup, select the Knight LLC contract created above.\n- Save.\n\nNotes:\n- No hot-keys were used; navigation was done via App Launcher search and lookups.\n- Dates entered: Contract 8/15/2025; Order 8/22/2025.\nSample 2:\nTASK:Prepare a contract for the account Lee Services. The start date is the third Monday of August, 2025. The contract term is 24 months. Then create an order based on this contract. The order start date should be the first working day of September, 2025.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Create the contract:\n - Open App Launcher > View All > Contracts, then click New.\n - Set Account to “Lee Services” (select the suggested record).\n - Set Start Date to 08/18/2025 (third Monday of August 2025).\n - Set Contract Term (months) to 24.\n - Save the contract.\n- Create the order based on the contract:\n - Open App Launcher > View All > Orders, then click New Order.\n - Set Account to “Lee Services” (select the record).\n - Set Order Start Date to 09/02/2025 (first working day of September 2025).\n - In Contract Number, select the Lee Services contract created above.\n - Save the order.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/215"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L454","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_contract_and_order","data_retrieval_spec":[{"type":"soql","id":"contract_creation","params":{"query":"SELECT Id, StartDate, ContractTerm FROM Contract WHERE Account.Name='{account_name}'"}},{"type":"soql","id":"order_creation","params":{"query":"SELECT Id, EffectiveDate FROM Order WHERE ContractId='{retrieved_results['contract_creation'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f1b9e5d4bf060eb4df19c95f04b8d5aac57a10eb1caf8259fa8ef7f71c709e06"} {"id":"scuba:sales_010_001","source_id":"scuba","original_id":"sales_010_001","kind":"benchmark_task","instruction":"Go ahead and spin up a Sales Process called 'B2B' with exactly these stages—and nothing else: Prospecting, Qualification, Closed Won, Closed Lost.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_001","query_templated":"Create a new Sales Process named 'B2B' which only includes the following stages: Prospecting, Qualification, Closed Won, Closed Lost.","query":"Go ahead and spin up a Sales Process called 'B2B' with exactly these stages—and nothing else: Prospecting, Qualification, Closed Won, Closed Lost.","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"B2B","comma_separated_stages_list":"Prospecting, Qualification, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"B2B","comma_separated_stages_list":"Prospecting, Qualification, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/216","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_010","template_name":"create_sales_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/216"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/216"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_sales_process"}],"source_schema_notes":[],"template_registry":{"name":"create_sales_process","original":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nGo ahead and spin up a Sales Process called 'B2B' with exactly these stages—and nothing else: Prospecting, Qualification, Closed Won, Closed Lost.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_001","query_templated":"Create a new Sales Process named 'B2B' which only includes the following stages: Prospecting, Qualification, Closed Won, Closed Lost.","query":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nGo ahead and spin up a Sales Process called 'B2B' with exactly these stages—and nothing else: Prospecting, Qualification, Closed Won, Closed Lost.\n```","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"B2B","comma_separated_stages_list":"Prospecting, Qualification, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"B2B","comma_separated_stages_list":"Prospecting, Qualification, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/216"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L285","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_sales_process","data_retrieval_spec":[{"type":"metadata","id":"sales_process_metadata","params":{"types_and_members":{"CustomObject":["Opportunity"]},"filepath":"businessProcesses/{sales_process_name.replace(' ', '_')}.businessProcess-meta.xml"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"da2b6e0124eb752d79be4b2f1cf4149dade9ff531c17f2b884bfee866aeccef6"} {"id":"scuba:sales_010_002","source_id":"scuba","original_id":"sales_010_002","kind":"benchmark_task","instruction":"Please configure a new Sales Process named 'SMB Sales' and restrict it solely to these stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_002","query_templated":"Create a new Sales Process named 'SMB Sales' which only includes the following stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.","query":"Please configure a new Sales Process named 'SMB Sales' and restrict it solely to these stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"SMB Sales","comma_separated_stages_list":"Prospecting, Value Proposition, Negotiation%2FReview, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"SMB Sales","comma_separated_stages_list":"Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/217","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_010","template_name":"create_sales_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/217"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/217"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_sales_process"}],"source_schema_notes":[],"template_registry":{"name":"create_sales_process","original":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nPlease configure a new Sales Process named 'SMB Sales' and restrict it solely to these stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_002","query_templated":"Create a new Sales Process named 'SMB Sales' which only includes the following stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.","query":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nPlease configure a new Sales Process named 'SMB Sales' and restrict it solely to these stages: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n```","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"SMB Sales","comma_separated_stages_list":"Prospecting, Value Proposition, Negotiation%2FReview, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"SMB Sales","comma_separated_stages_list":"Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/217"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L285","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_sales_process","data_retrieval_spec":[{"type":"metadata","id":"sales_process_metadata","params":{"types_and_members":{"CustomObject":["Opportunity"]},"filepath":"businessProcesses/{sales_process_name.replace(' ', '_')}.businessProcess-meta.xml"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9c5139da2502d2b3b3e84610c54b465287264ad86aa6d7fa80b73b8ad1dc7247"} {"id":"scuba:sales_010_003","source_id":"scuba","original_id":"sales_010_003","kind":"benchmark_task","instruction":"Let’s keep this simple: create a Sales Process named 'B2C' that includes only three stages—Needs Analysis, Proposal/Price Quote, and Closed Won.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_003","query_templated":"Create a new Sales Process named 'B2C' which only includes the following stages: Needs Analysis, Proposal/Price Quote, Closed Won.","query":"Let’s keep this simple: create a Sales Process named 'B2C' that includes only three stages—Needs Analysis, Proposal/Price Quote, and Closed Won.","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"B2C","comma_separated_stages_list":"Needs Analysis, Proposal%2FPrice Quote, Closed Won"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"B2C","comma_separated_stages_list":"Needs Analysis, Proposal/Price Quote, Closed Won"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/218","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_010","template_name":"create_sales_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/218"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/218"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_sales_process"}],"source_schema_notes":[],"template_registry":{"name":"create_sales_process","original":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nLet’s keep this simple: create a Sales Process named 'B2C' that includes only three stages—Needs Analysis, Proposal/Price Quote, and Closed Won.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_003","query_templated":"Create a new Sales Process named 'B2C' which only includes the following stages: Needs Analysis, Proposal/Price Quote, Closed Won.","query":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nLet’s keep this simple: create a Sales Process named 'B2C' that includes only three stages—Needs Analysis, Proposal/Price Quote, and Closed Won.\n```","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"B2C","comma_separated_stages_list":"Needs Analysis, Proposal%2FPrice Quote, Closed Won"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"B2C","comma_separated_stages_list":"Needs Analysis, Proposal/Price Quote, Closed Won"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/218"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L285","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_sales_process","data_retrieval_spec":[{"type":"metadata","id":"sales_process_metadata","params":{"types_and_members":{"CustomObject":["Opportunity"]},"filepath":"businessProcesses/{sales_process_name.replace(' ', '_')}.businessProcess-meta.xml"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d1ada93b1db4361803414d47c366aaccc8edec636c961e47a4549f47a6eda243"} {"id":"scuba:sales_010_004","source_id":"scuba","original_id":"sales_010_004","kind":"benchmark_task","instruction":"Could you set up a Sales Process titled 'Customer Journey Sales Process' and make sure it contains just these stages, no others: Prospecting, Value Proposition, Closed Won, Closed Lost?","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_004","query_templated":"Create a new Sales Process named 'Customer Journey Sales Process' which only includes the following stages: Prospecting, Value Proposition, Closed Won, Closed Lost.","query":"Could you set up a Sales Process titled 'Customer Journey Sales Process' and make sure it contains just these stages, no others: Prospecting, Value Proposition, Closed Won, Closed Lost?","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"Customer Journey Sales Process","comma_separated_stages_list":"Prospecting, Value Proposition, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"Customer Journey Sales Process","comma_separated_stages_list":"Prospecting, Value Proposition, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/219","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_010","template_name":"create_sales_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/219"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/219"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_sales_process"}],"source_schema_notes":[],"template_registry":{"name":"create_sales_process","original":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nCould you set up a Sales Process titled 'Customer Journey Sales Process' and make sure it contains just these stages, no others: Prospecting, Value Proposition, Closed Won, Closed Lost?\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_004","query_templated":"Create a new Sales Process named 'Customer Journey Sales Process' which only includes the following stages: Prospecting, Value Proposition, Closed Won, Closed Lost.","query":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nCould you set up a Sales Process titled 'Customer Journey Sales Process' and make sure it contains just these stages, no others: Prospecting, Value Proposition, Closed Won, Closed Lost?\n```","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"Customer Journey Sales Process","comma_separated_stages_list":"Prospecting, Value Proposition, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"Customer Journey Sales Process","comma_separated_stages_list":"Prospecting, Value Proposition, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/219"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L285","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_sales_process","data_retrieval_spec":[{"type":"metadata","id":"sales_process_metadata","params":{"types_and_members":{"CustomObject":["Opportunity"]},"filepath":"businessProcesses/{sales_process_name.replace(' ', '_')}.businessProcess-meta.xml"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"afd631a9edf5fe1f09d95a141258b12f8078ed16f1993153217ce9207defdd06"} {"id":"scuba:sales_010_005","source_id":"scuba","original_id":"sales_010_005","kind":"benchmark_task","instruction":"Kick off a new Sales Process called 'Rocket Launch Sales Methodology' and limit it to only these stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_005","query_templated":"Create a new Sales Process named 'Rocket Launch Sales Methodology' which only includes the following stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.","query":"Kick off a new Sales Process called 'Rocket Launch Sales Methodology' and limit it to only these stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"Rocket Launch Sales Methodology","comma_separated_stages_list":"Proposal%2FPrice Quote, Negotiation%2FReview, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"Rocket Launch Sales Methodology","comma_separated_stages_list":"Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/220","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_010","template_name":"create_sales_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/220"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/220"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_sales_process"}],"source_schema_notes":[],"template_registry":{"name":"create_sales_process","original":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_sales_process","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nKick off a new Sales Process called 'Rocket Launch Sales Methodology' and limit it to only these stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_010_005","query_templated":"Create a new Sales Process named 'Rocket Launch Sales Methodology' which only includes the following stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.","query":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process.\n\n## Your ultimate task\n```\nKick off a new Sales Process called 'Rocket Launch Sales Methodology' and limit it to only these stages: Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost.\n```","human_review_comment":"accept","ground_truth_dict":{"sales_process_name":"Rocket Launch Sales Methodology","comma_separated_stages_list":"Proposal%2FPrice Quote, Negotiation%2FReview, Closed Won, Closed Lost"},"difficulty":"easy","query_template_name":"create_sales_process","query_template_metadata":{"template_id":"sales_010","template_string":"Create a new Sales Process named '{sales_process_name}' which only includes the following stages: {comma_separated_stages_list}.","prerequisites":{},"objects":[],"metadata_types":["BusinessProcess"]},"instance_dict":{"sales_process_name":"Rocket Launch Sales Methodology","comma_separated_stages_list":"Proposal/Price Quote, Negotiation/Review, Closed Won, Closed Lost"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:I want to create a new Sales Process named \"B2B\", which only includes the \"Prospecting\", Qualification\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- Navigate: Feature Settings > Sales > Sales Processes.\n- Click New Sales Process.\n- Enter Sales Process Name: B2B, then click Save to open the stage selection page.\n- Configure included stages by removing all unwanted stages so only these remain: Prospecting, Qualification, Closed Won, Closed Lost.\n - Remove: Needs Analysis, Value Proposition, Id. Decision Makers, Perception Analysis, Proposal/Price Quote, Negotiation/Review.\n- Click Save to finalize the B2B sales process.\nSample 2:\nTASK:Our small to medium-sized business team needs a new Sales Process called \"SMB Sales\", which only includes the \"Prospecting\", \"Value Proposition\", \"Negotiation/Review\", \"Closed Won\", and \"Closed Lost\" values.\nEXPERIENCE:\nSuccessful plan to create the “SMB Sales” Sales Process with only the required stages:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Sales Processes” and open it.\n- Click “New” to create a Sales Process.\n- Enter “SMB Sales” as the Sales Process Name and click Save.\n- In the stage selection screen, keep only these values: Prospecting, Value Proposition, Negotiation/Review, Closed Won, Closed Lost.\n - Remove the following values:\n - Qualification\n - Needs Analysis\n - Id. Decision Makers\n - Perception Analysis\n - Proposal/Price Quote\n- Click Save to finalize the Sales Process."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/220"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L285","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_sales_process","data_retrieval_spec":[{"type":"metadata","id":"sales_process_metadata","params":{"types_and_members":{"CustomObject":["Opportunity"]},"filepath":"businessProcesses/{sales_process_name.replace(' ', '_')}.businessProcess-meta.xml"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"04f06e6ce5c4a2b663589f88c70e50d6ecf5285f0e8a7e52940553594a886274"} {"id":"scuba:sales_011_001","source_id":"scuba","original_id":"sales_011_001","kind":"benchmark_task","instruction":"Spin up a new territory model called 'by revenue'. Give its rule the name 'annual revenue is USD'. Filter the selection to accounts where annual revenue is greater than 10,000. If sales territory isn’t enabled yet, please enable it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_001","query_templated":"Create a territory model named 'by revenue' with rule name 'annual revenue is USD'. Set the selection criteria as 'annual revenue is greater than 10,000'. Enable sales territory if not already enabled.","query":"Spin up a new territory model called 'by revenue'. Give its rule the name 'annual revenue is USD'. Filter the selection to accounts where annual revenue is greater than 10,000. If sales territory isn’t enabled yet, please enable it.","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by revenue","rule_name":"annual revenue is USD","filters":[["Account.AnnualRevenue","greaterThan",10000]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by revenue","rule_name":"annual revenue is USD","selection_criteria":"annual revenue is greater than 10,000"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/221","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_011","template_name":"create_territory_model","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/221"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/221"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_territory_model"}],"source_schema_notes":[],"template_registry":{"name":"create_territory_model","original":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nSpin up a new territory model called 'by revenue'. Give its rule the name 'annual revenue is USD'. Filter the selection to accounts where annual revenue is greater than 10,000. If sales territory isn’t enabled yet, please enable it.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_001","query_templated":"Create a territory model named 'by revenue' with rule name 'annual revenue is USD'. Set the selection criteria as 'annual revenue is greater than 10,000'. Enable sales territory if not already enabled.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nSpin up a new territory model called 'by revenue'. Give its rule the name 'annual revenue is USD'. Filter the selection to accounts where annual revenue is greater than 10,000. If sales territory isn’t enabled yet, please enable it.\n```","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by revenue","rule_name":"annual revenue is USD","filters":[["Account.AnnualRevenue","greaterThan",10000]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by revenue","rule_name":"annual revenue is USD","selection_criteria":"annual revenue is greater than 10,000"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/221"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L249","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_territory_model","data_retrieval_spec":[{"type":"soql","id":"territory_model_info","params":{"query":"SELECT Id FROM Territory2Model WHERE Name='{territory_model_name}'"}},{"type":"soql","id":"territory_rule_info","params":{"query":"SELECT Id, DeveloperName FROM ObjectTerritory2AssignmentRule WHERE MasterLabel='{rule_name}' AND Territory2ModelId='{retrieved_results['territory_model_info'].records[0]['Id']}'"}},{"type":"soql","id":"filters","params":{"query":"SELECT Field, Operation, Value FROM ObjectTerritory2AssignmentRuleItem WHERE RuleId='{retrieved_results['territory_rule_info'].records[0]['Id']}'"}},{"type":"metadata","id":"sales_territory_settings","params":{"types_and_members":{"Settings":["Territory2"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"802c9af3ddc06c3ab438d57e7ddcaaae8626f287d9b8c1626c31d0adc410fc21"} {"id":"scuba:sales_011_002","source_id":"scuba","original_id":"sales_011_002","kind":"benchmark_task","instruction":"Could you set up a territory model named 'by Employees' with the rule name 'Number of Employees and Year'? Apply this filter: total employees is less than or equal to 50 and account year started is greater than or equal to 2020. Also, make sure sales territory is turned on if it isn’t already.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_002","query_templated":"Create a territory model named 'by Employees' with rule name 'Number of Employees and Year'. Set the selection criteria as 'total employees is less than or equal to 50 and account year started is greater than or equal to 2020'. Enable sales territory if not already enabled.","query":"Could you set up a territory model named 'by Employees' with the rule name 'Number of Employees and Year'? Apply this filter: total employees is less than or equal to 50 and account year started is greater than or equal to 2020. Also, make sure sales territory is turned on if it isn’t already.","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by Employees","rule_name":"Number of Employees and Year","filters":[["Account.NumberOfEmployees","lessOrEqual",50],["Account.YearStarted","greaterOrEqual",2020]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by Employees","rule_name":"Number of Employees and Year","selection_criteria":"total employees is less than or equal to 50 and account year started is greater than or equal to 2020"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/222","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_011","template_name":"create_territory_model","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/222"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/222"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_territory_model"}],"source_schema_notes":[],"template_registry":{"name":"create_territory_model","original":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nCould you set up a territory model named 'by Employees' with the rule name 'Number of Employees and Year'? Apply this filter: total employees is less than or equal to 50 and account year started is greater than or equal to 2020. Also, make sure sales territory is turned on if it isn’t already.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_002","query_templated":"Create a territory model named 'by Employees' with rule name 'Number of Employees and Year'. Set the selection criteria as 'total employees is less than or equal to 50 and account year started is greater than or equal to 2020'. Enable sales territory if not already enabled.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nCould you set up a territory model named 'by Employees' with the rule name 'Number of Employees and Year'? Apply this filter: total employees is less than or equal to 50 and account year started is greater than or equal to 2020. Also, make sure sales territory is turned on if it isn’t already.\n```","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by Employees","rule_name":"Number of Employees and Year","filters":[["Account.NumberOfEmployees","lessOrEqual",50],["Account.YearStarted","greaterOrEqual",2020]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by Employees","rule_name":"Number of Employees and Year","selection_criteria":"total employees is less than or equal to 50 and account year started is greater than or equal to 2020"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/222"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L249","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_territory_model","data_retrieval_spec":[{"type":"soql","id":"territory_model_info","params":{"query":"SELECT Id FROM Territory2Model WHERE Name='{territory_model_name}'"}},{"type":"soql","id":"territory_rule_info","params":{"query":"SELECT Id, DeveloperName FROM ObjectTerritory2AssignmentRule WHERE MasterLabel='{rule_name}' AND Territory2ModelId='{retrieved_results['territory_model_info'].records[0]['Id']}'"}},{"type":"soql","id":"filters","params":{"query":"SELECT Field, Operation, Value FROM ObjectTerritory2AssignmentRuleItem WHERE RuleId='{retrieved_results['territory_rule_info'].records[0]['Id']}'"}},{"type":"metadata","id":"sales_territory_settings","params":{"types_and_members":{"Settings":["Territory2"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"416bf56c17241179b723121ac8c968f264d6df4b1648c8a5f24e9107e0aa3a2b"} {"id":"scuba:sales_011_003","source_id":"scuba","original_id":"sales_011_003","kind":"benchmark_task","instruction":"Please ensure sales territory is enabled. Then create a territory model titled 'by Finance Industry' and assign the rule name 'Industry, Active and Type'. Configure the criteria so that Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_003","query_templated":"Create a territory model named 'by Finance Industry' with rule name 'Industry, Active and Type'. Set the selection criteria as 'Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner'. Enable sales territory if not already enabled.","query":"Please ensure sales territory is enabled. Then create a territory model titled 'by Finance Industry' and assign the rule name 'Industry, Active and Type'. Configure the criteria so that Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner.","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by Finance Industry","rule_name":"Industry, Active and Type","filters":[["Account.Industry","equals","Finance"],["Account.Active__c","equals","Yes"],["Account.Type","notEqual","Channel Partner / Reseller"],["Account.Type","notEqual","Installation Partner"]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by Finance Industry","rule_name":"Industry, Active and Type","selection_criteria":"Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/223","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_011","template_name":"create_territory_model","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/223"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/223"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_territory_model"}],"source_schema_notes":[],"template_registry":{"name":"create_territory_model","original":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nPlease ensure sales territory is enabled. Then create a territory model titled 'by Finance Industry' and assign the rule name 'Industry, Active and Type'. Configure the criteria so that Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_003","query_templated":"Create a territory model named 'by Finance Industry' with rule name 'Industry, Active and Type'. Set the selection criteria as 'Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner'. Enable sales territory if not already enabled.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nPlease ensure sales territory is enabled. Then create a territory model titled 'by Finance Industry' and assign the rule name 'Industry, Active and Type'. Configure the criteria so that Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner.\n```","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"by Finance Industry","rule_name":"Industry, Active and Type","filters":[["Account.Industry","equals","Finance"],["Account.Active__c","equals","Yes"],["Account.Type","notEqual","Channel Partner / Reseller"],["Account.Type","notEqual","Installation Partner"]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"by Finance Industry","rule_name":"Industry, Active and Type","selection_criteria":"Industry equals Finance, Account equals Active, and Type is not equal to Channel Partner / Reseller or Installation Partner"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/223"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L249","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_territory_model","data_retrieval_spec":[{"type":"soql","id":"territory_model_info","params":{"query":"SELECT Id FROM Territory2Model WHERE Name='{territory_model_name}'"}},{"type":"soql","id":"territory_rule_info","params":{"query":"SELECT Id, DeveloperName FROM ObjectTerritory2AssignmentRule WHERE MasterLabel='{rule_name}' AND Territory2ModelId='{retrieved_results['territory_model_info'].records[0]['Id']}'"}},{"type":"soql","id":"filters","params":{"query":"SELECT Field, Operation, Value FROM ObjectTerritory2AssignmentRuleItem WHERE RuleId='{retrieved_results['territory_rule_info'].records[0]['Id']}'"}},{"type":"metadata","id":"sales_territory_settings","params":{"types_and_members":{"Settings":["Territory2"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8c1662037b6a3e591df0517dc74a34f1cff07fae6d40fd7a9aadc4a7720eb8af"} {"id":"scuba:sales_011_004","source_id":"scuba","original_id":"sales_011_004","kind":"benchmark_task","instruction":"If sales territory is disabled, enable it. After that, create a territory model named 'Global Sales Territory' with a rule also called 'Global Sales Territory'. Set the selection criteria to include only records where Customer Priority is High.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_004","query_templated":"Create a territory model named 'Global Sales Territory' with rule name 'Global Sales Territory'. Set the selection criteria as 'Customer Priority is High'. Enable sales territory if not already enabled.","query":"If sales territory is disabled, enable it. After that, create a territory model named 'Global Sales Territory' with a rule also called 'Global Sales Territory'. Set the selection criteria to include only records where Customer Priority is High.","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"Global Sales Territory","rule_name":"Global Sales Territory","filters":[["Account.CustomerPriority__c","equals","High"]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"Global Sales Territory","rule_name":"Global Sales Territory","selection_criteria":"Customer Priority is High"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/224","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Configuration","source_difficulty":"easy","template_id":"sales_011","template_name":"create_territory_model","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/224"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/224"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_territory_model"}],"source_schema_notes":[],"template_registry":{"name":"create_territory_model","original":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_territory_model","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nIf sales territory is disabled, enable it. After that, create a territory model named 'Global Sales Territory' with a rule also called 'Global Sales Territory'. Set the selection criteria to include only records where Customer Priority is High.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Configuration","task_id":"sales_011_004","query_templated":"Create a territory model named 'Global Sales Territory' with rule name 'Global Sales Territory'. Set the selection criteria as 'Customer Priority is High'. Enable sales territory if not already enabled.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled).\n\n## Your ultimate task\n```\nIf sales territory is disabled, enable it. After that, create a territory model named 'Global Sales Territory' with a rule also called 'Global Sales Territory'. Set the selection criteria to include only records where Customer Priority is High.\n```","human_review_comment":"accept","ground_truth_dict":{"territory_model_name":"Global Sales Territory","rule_name":"Global Sales Territory","filters":[["Account.CustomerPriority__c","equals","High"]]},"difficulty":"easy","query_template_name":"create_territory_model","query_template_metadata":{"template_id":"sales_011","template_string":"Create a territory model named '{territory_model_name}' with rule name '{rule_name}'. Set the selection criteria as '{selection_criteria}'. Enable sales territory if not already enabled.","prerequisites":{},"objects":["ObjectTerritory2AssignmentRuleItem","ObjectTerritory2AssignmentRule","Territory2Model"],"metadata_types":[]},"instance_dict":{"territory_model_name":"Global Sales Territory","rule_name":"Global Sales Territory","selection_criteria":"Customer Priority is High"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a territory model with the name \"by revenue\" and the rule name \"annual revenue is USD\". The selection criteria is \"annual revenue is greater than 10,000\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Feature Settings > Sales > Territories > Territory Settings, and enable Sales Territory (if not already enabled).\n- Use Quick Find to search for “Territory Models” and open it.\n- Create a new territory model:\n - Click New Territory Model.\n - Set Territory Model Label to “by revenue”.\n - Save.\n- Create the rule:\n - Click View Rules, then New Rule.\n - Set Rule Name to “annual revenue is USD”.\n - Set Field to “Account: Annual Revenue”, Operator to “greater than”, and Value to “10000”.\n - Check Set as Active.\n - Save.\nSample 2:\nTASK:Create a territory model with the name \"by Employees\" and with the rule name \"Number of Employees and Year\". The selection criteria is \"total employees is less than or equal to 50\" and \"account year started is greater than or equal to 2020\". If the sales territory is not enabled, enable it first.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- In Quick Find, type “Territory” and open “Territory Models.”\n- Create the model:\n - Click “New Territory Model.”\n - Enter label: “by Employees.”\n - Click “Save.”\n- Add the rule:\n - Click “View Rules,” then “New Rule.”\n - Enter rule name: “Number of Employees and Year.”\n - Add condition 1: Field = “Account: Employees”; Operator = “less or equal”; Value = “50.”\n - Add condition 2: Field = “Account: Year”; Operator = “greater or equal”; Value = “2020.”\n - Check “Set as Active.”\n - Click “Save.”\n\nNotes:\n- No hotkeys were used.\n- Enabling Sales Territories was not required in this run (already enabled)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/224"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L249","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_territory_model","data_retrieval_spec":[{"type":"soql","id":"territory_model_info","params":{"query":"SELECT Id FROM Territory2Model WHERE Name='{territory_model_name}'"}},{"type":"soql","id":"territory_rule_info","params":{"query":"SELECT Id, DeveloperName FROM ObjectTerritory2AssignmentRule WHERE MasterLabel='{rule_name}' AND Territory2ModelId='{retrieved_results['territory_model_info'].records[0]['Id']}'"}},{"type":"soql","id":"filters","params":{"query":"SELECT Field, Operation, Value FROM ObjectTerritory2AssignmentRuleItem WHERE RuleId='{retrieved_results['territory_rule_info'].records[0]['Id']}'"}},{"type":"metadata","id":"sales_territory_settings","params":{"types_and_members":{"Settings":["Territory2"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"8f1997af6c77d4787db85ed3c3942cbc468bfd5c5f9af82589d50548c9587ffe"} {"id":"scuba:sales_012_001","source_id":"scuba","original_id":"sales_012_001","kind":"benchmark_task","instruction":"Please create a new campaign called \"Introduce the new shoes collections\" as a Webinar. Schedule it to start two weeks from today. Set the budget at 10,000 and aim for expected revenue equal to 10x the budget. Invite Ali Alaba and Leo Messi.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_001","query_templated":"Create a new campaign with following details - Name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10000; Expected Revenue in Campaign: 10 times the budget. Finally, invite the following leads to this campaign: Ali Alaba and Leo Messi","query":"Please create a new campaign called \"Introduce the new shoes collections\" as a Webinar. Schedule it to start two weeks from today. Set the budget at 10,000 and aim for expected revenue equal to 10x the budget. Invite Ali Alaba and Leo Messi.","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Introduce the new shoes collections","campaign_type":"Webinar","start_date":2,"budget_amount":10000,"expected_revenue":100000,"leads":["Ali Alaba","Leo Messi"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Introduce the new shoes collections","campaign_type":"Webinar","start_date":"two weeks from today","budget_amount":10000,"expected_revenue":"10 times the budget","leads":"Ali Alaba and Leo Messi"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/225","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Campaign","source_difficulty":"easy","template_id":"sales_012","template_name":"create_campaign","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/225"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/225"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_campaign"}],"source_schema_notes":[],"template_registry":{"name":"create_campaign","original":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a new campaign called \"Introduce the new shoes collections\" as a Webinar. Schedule it to start two weeks from today. Set the budget at 10,000 and aim for expected revenue equal to 10x the budget. Invite Ali Alaba and Leo Messi.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_001","query_templated":"Create a new campaign with following details - Name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10000; Expected Revenue in Campaign: 10 times the budget. Finally, invite the following leads to this campaign: Ali Alaba and Leo Messi","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease create a new campaign called \"Introduce the new shoes collections\" as a Webinar. Schedule it to start two weeks from today. Set the budget at 10,000 and aim for expected revenue equal to 10x the budget. Invite Ali Alaba and Leo Messi.\n```","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Introduce the new shoes collections","campaign_type":"Webinar","start_date":2,"budget_amount":10000,"expected_revenue":100000,"leads":["Ali Alaba","Leo Messi"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Introduce the new shoes collections","campaign_type":"Webinar","start_date":"two weeks from today","budget_amount":10000,"expected_revenue":"10 times the budget","leads":"Ali Alaba and Leo Messi"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/225"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L491","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_campaign","data_retrieval_spec":[{"type":"soql","id":"campaign_info","params":{"query":"SELECT Id, Type, StartDate, BudgetedCost, ExpectedRevenue FROM Campaign WHERE Name='{campaign_name}'"}},{"type":"soql","id":"campaign_members","params":{"query":"SELECT Lead.Name FROM CampaignMember WHERE Campaign.Name='{campaign_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d383f2e0ed4a1077037caa2b7a3d196e94ef322bc6189f1730e42f413ebf03a5"} {"id":"scuba:sales_012_002","source_id":"scuba","original_id":"sales_012_002","kind":"benchmark_task","instruction":"Set up a campaign with these details: - Name: Autumn Trail Running Launch - Type: Direct Mail - Start date: four weeks from today - Budget: 10,000 - Expected revenue: 3 times the budget Then invite Amanda Klein to the campaign.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_002","query_templated":"Create a new campaign with following details - Name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10000; Expected Revenue in Campaign: 3 times the Budget. Finally, invite the following leads to this campaign: Amanda Klein","query":"Set up a campaign with these details: - Name: Autumn Trail Running Launch - Type: Direct Mail - Start date: four weeks from today - Budget: 10,000 - Expected revenue: 3 times the budget Then invite Amanda Klein to the campaign.","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Autumn Trail Running Launch","campaign_type":"Direct Mail","start_date":4,"budget_amount":10000,"expected_revenue":30000,"leads":["Amanda Klein"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Autumn Trail Running Launch","campaign_type":"Direct Mail","start_date":"four weeks from today","budget_amount":10000,"expected_revenue":"3 times the Budget","leads":"Amanda Klein"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/226","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Campaign","source_difficulty":"easy","template_id":"sales_012","template_name":"create_campaign","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/226"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/226"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_campaign"}],"source_schema_notes":[],"template_registry":{"name":"create_campaign","original":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nSet up a campaign with these details: - Name: Autumn Trail Running Launch - Type: Direct Mail - Start date: four weeks from today - Budget: 10,000 - Expected revenue: 3 times the budget Then invite Amanda Klein to the campaign.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_002","query_templated":"Create a new campaign with following details - Name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10000; Expected Revenue in Campaign: 3 times the Budget. Finally, invite the following leads to this campaign: Amanda Klein","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nSet up a campaign with these details: - Name: Autumn Trail Running Launch - Type: Direct Mail - Start date: four weeks from today - Budget: 10,000 - Expected revenue: 3 times the budget Then invite Amanda Klein to the campaign.\n```","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Autumn Trail Running Launch","campaign_type":"Direct Mail","start_date":4,"budget_amount":10000,"expected_revenue":30000,"leads":["Amanda Klein"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Autumn Trail Running Launch","campaign_type":"Direct Mail","start_date":"four weeks from today","budget_amount":10000,"expected_revenue":"3 times the Budget","leads":"Amanda Klein"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/226"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L491","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_campaign","data_retrieval_spec":[{"type":"soql","id":"campaign_info","params":{"query":"SELECT Id, Type, StartDate, BudgetedCost, ExpectedRevenue FROM Campaign WHERE Name='{campaign_name}'"}},{"type":"soql","id":"campaign_members","params":{"query":"SELECT Lead.Name FROM CampaignMember WHERE Campaign.Name='{campaign_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"552953850e1c81c5373f9b0bb50be45ee58be6eb5521b159a1350f0637a68cd4"} {"id":"scuba:sales_012_003","source_id":"scuba","original_id":"sales_012_003","kind":"benchmark_task","instruction":"Could you launch an Email campaign titled \"Summer Yoga Webinar Series\" starting one week from today? Budget it at 9,000, and set expected revenue to four times the budget plus 1,500. Please invite Priya Sharma, Lucas Chen, and Jordan Hayes.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_003","query_templated":"Create a new campaign with following details - Name: Summer Yoga Webinar Series; type: Email; Start Date: one week from today; Budget: 9000; Expected Revenue in Campaign: 4 times the Budget plus 1,500. Finally, invite the following leads to this campaign: Priya Sharma, Lucas Chen, Jordan Hayes","query":"Could you launch an Email campaign titled \"Summer Yoga Webinar Series\" starting one week from today? Budget it at 9,000, and set expected revenue to four times the budget plus 1,500. Please invite Priya Sharma, Lucas Chen, and Jordan Hayes.","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Summer Yoga Webinar Series","campaign_type":"Email","start_date":1,"budget_amount":9000,"expected_revenue":39500,"leads":["Priya Sharma","Lucas Chen","Jordan Hayes"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Summer Yoga Webinar Series","campaign_type":"Email","start_date":"one week from today","budget_amount":9000,"expected_revenue":"4 times the Budget plus 1,500","leads":"Priya Sharma, Lucas Chen, Jordan Hayes"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/227","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Campaign","source_difficulty":"easy","template_id":"sales_012","template_name":"create_campaign","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/227"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/227"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_campaign"}],"source_schema_notes":[],"template_registry":{"name":"create_campaign","original":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you launch an Email campaign titled \"Summer Yoga Webinar Series\" starting one week from today? Budget it at 9,000, and set expected revenue to four times the budget plus 1,500. Please invite Priya Sharma, Lucas Chen, and Jordan Hayes.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_003","query_templated":"Create a new campaign with following details - Name: Summer Yoga Webinar Series; type: Email; Start Date: one week from today; Budget: 9000; Expected Revenue in Campaign: 4 times the Budget plus 1,500. Finally, invite the following leads to this campaign: Priya Sharma, Lucas Chen, Jordan Hayes","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCould you launch an Email campaign titled \"Summer Yoga Webinar Series\" starting one week from today? Budget it at 9,000, and set expected revenue to four times the budget plus 1,500. Please invite Priya Sharma, Lucas Chen, and Jordan Hayes.\n```","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Summer Yoga Webinar Series","campaign_type":"Email","start_date":1,"budget_amount":9000,"expected_revenue":39500,"leads":["Priya Sharma","Lucas Chen","Jordan Hayes"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Summer Yoga Webinar Series","campaign_type":"Email","start_date":"one week from today","budget_amount":9000,"expected_revenue":"4 times the Budget plus 1,500","leads":"Priya Sharma, Lucas Chen, Jordan Hayes"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/227"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L491","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_campaign","data_retrieval_spec":[{"type":"soql","id":"campaign_info","params":{"query":"SELECT Id, Type, StartDate, BudgetedCost, ExpectedRevenue FROM Campaign WHERE Name='{campaign_name}'"}},{"type":"soql","id":"campaign_members","params":{"query":"SELECT Lead.Name FROM CampaignMember WHERE Campaign.Name='{campaign_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5b7de6e764f100aa73a308178ecb6d80cd022eb9877f546e7e64ab6afcfc5ddc"} {"id":"scuba:sales_012_004","source_id":"scuba","original_id":"sales_012_004","kind":"benchmark_task","instruction":"Go ahead and create a campaign named \"Spring Sale Campaign\" as a Conference. Kickoff should be three weeks from today. Allocate a 5,000 budget and project expected revenue at triple the budget. Invite Charlie Green and Emily White.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_004","query_templated":"Create a new campaign with following details - Name: Spring Sale Campaign; type: Conference; Start Date: three weeks from today; Budget: 5000; Expected Revenue in Campaign: triple the budget. Finally, invite the following leads to this campaign: Charlie Green and Emily White","query":"Go ahead and create a campaign named \"Spring Sale Campaign\" as a Conference. Kickoff should be three weeks from today. Allocate a 5,000 budget and project expected revenue at triple the budget. Invite Charlie Green and Emily White.","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Spring Sale Campaign","campaign_type":"Conference","start_date":3,"budget_amount":5000,"expected_revenue":15000,"leads":["Charlie Green","Emily White"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Spring Sale Campaign","campaign_type":"Conference","start_date":"three weeks from today","budget_amount":5000,"expected_revenue":"triple the budget","leads":"Charlie Green and Emily White"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/228","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Campaign","source_difficulty":"easy","template_id":"sales_012","template_name":"create_campaign","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/228"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/228"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_campaign"}],"source_schema_notes":[],"template_registry":{"name":"create_campaign","original":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGo ahead and create a campaign named \"Spring Sale Campaign\" as a Conference. Kickoff should be three weeks from today. Allocate a 5,000 budget and project expected revenue at triple the budget. Invite Charlie Green and Emily White.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_004","query_templated":"Create a new campaign with following details - Name: Spring Sale Campaign; type: Conference; Start Date: three weeks from today; Budget: 5000; Expected Revenue in Campaign: triple the budget. Finally, invite the following leads to this campaign: Charlie Green and Emily White","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGo ahead and create a campaign named \"Spring Sale Campaign\" as a Conference. Kickoff should be three weeks from today. Allocate a 5,000 budget and project expected revenue at triple the budget. Invite Charlie Green and Emily White.\n```","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Spring Sale Campaign","campaign_type":"Conference","start_date":3,"budget_amount":5000,"expected_revenue":15000,"leads":["Charlie Green","Emily White"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Spring Sale Campaign","campaign_type":"Conference","start_date":"three weeks from today","budget_amount":5000,"expected_revenue":"triple the budget","leads":"Charlie Green and Emily White"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/228"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L491","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_campaign","data_retrieval_spec":[{"type":"soql","id":"campaign_info","params":{"query":"SELECT Id, Type, StartDate, BudgetedCost, ExpectedRevenue FROM Campaign WHERE Name='{campaign_name}'"}},{"type":"soql","id":"campaign_members","params":{"query":"SELECT Lead.Name FROM CampaignMember WHERE Campaign.Name='{campaign_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d3b16c116438b0ff41bbc4f76e630f5f70dac119102e85650034485227468fd3"} {"id":"scuba:sales_012_005","source_id":"scuba","original_id":"sales_012_005","kind":"benchmark_task","instruction":"Let’s go big—set up a Trade Show campaign called \"Summer Tech Innovation Showcase\" that starts eight weeks from today. Budget is 75,000, and the expected revenue should be three times the investment. Add Grace King and Frank Knight to the invite list.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_005","query_templated":"Create a new campaign with following details - Name: Summer Tech Innovation Showcase; type: Trade Show; Start Date: eight weeks from today; Budget: 75000; Expected Revenue in Campaign: triple the investment. Finally, invite the following leads to this campaign: Grace King and Frank Knight","query":"Let’s go big—set up a Trade Show campaign called \"Summer Tech Innovation Showcase\" that starts eight weeks from today. Budget is 75,000, and the expected revenue should be three times the investment. Add Grace King and Frank Knight to the invite list.","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Summer Tech Innovation Showcase","campaign_type":"Trade Show","start_date":8,"budget_amount":75000,"expected_revenue":225000,"leads":["Grace King","Frank Knight"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Summer Tech Innovation Showcase","campaign_type":"Trade Show","start_date":"eight weeks from today","budget_amount":75000,"expected_revenue":"triple the investment","leads":"Grace King and Frank Knight"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/229","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"Campaign","source_difficulty":"easy","template_id":"sales_012","template_name":"create_campaign","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/229"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/229"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","sha256":"8b65ec249f5e093c788c492ea85150cbfa2608e850ae779cca9f61630dd121ba","source_json_pointer":"/create_campaign"}],"source_schema_notes":[],"template_registry":{"name":"create_campaign","original":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_sales.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_sales.json","sha256":"492d2edf81efe5da5beb73adbd6b1a43edd2b4ec5f099b75f02d8ce34dabc19a","source_json_pointer":"/create_campaign","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s go big—set up a Trade Show campaign called \"Summer Tech Innovation Showcase\" that starts eight weeks from today. Budget is 75,000, and the expected revenue should be three times the investment. Add Grace King and Frank Knight to the invite list.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"sales","subcategory":"Campaign","task_id":"sales_012_005","query_templated":"Create a new campaign with following details - Name: Summer Tech Innovation Showcase; type: Trade Show; Start Date: eight weeks from today; Budget: 75000; Expected Revenue in Campaign: triple the investment. Finally, invite the following leads to this campaign: Grace King and Frank Knight","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nLet’s go big—set up a Trade Show campaign called \"Summer Tech Innovation Showcase\" that starts eight weeks from today. Budget is 75,000, and the expected revenue should be three times the investment. Add Grace King and Frank Knight to the invite list.\n```","human_review_comment":"accept","ground_truth_dict":{"campaign_name":"Summer Tech Innovation Showcase","campaign_type":"Trade Show","start_date":8,"budget_amount":75000,"expected_revenue":225000,"leads":["Grace King","Frank Knight"]},"difficulty":"easy","query_template_name":"create_campaign","query_template_metadata":{"template_id":"sales_012","template_string":"Create a new campaign with following details - Name: {campaign_name}; type: {campaign_type}; Start Date: {start_date}; Budget: {budget_amount}; Expected Revenue in Campaign: {expected_revenue}. Finally, invite the following leads to this campaign: {leads}","prerequisites":{"data":{"objects":["Lead"]}},"objects":["Campaign"],"metadata_types":["ValidationRule"]},"instance_dict":{"campaign_name":"Summer Tech Innovation Showcase","campaign_type":"Trade Show","start_date":"eight weeks from today","budget_amount":75000,"expected_revenue":"triple the investment","leads":"Grace King and Frank Knight"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a new campaign with following details. [name: Introduce the new shoes collections; type: Webinar; Start Date: two weeks from today; Budget: 10,000; Expected Revenue in Campaign: 10 times the Budget.] Finally, invite the following leads to this campaign: [Ali Alaba and Leo Messi]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite leads:\n\n1) Log in and navigate to create a new campaign\n- Click “Log In”\n- Click “Campaigns” > “New”\n\n2) Enter campaign details\n- Campaign Name: “Introduce the new shoes collections”\n- Type: Select “Webinar” from the Type dropdown\n- Start Date: Set to two weeks from today (entered as 09/01/2025 in the run)\n- Budget cost in Campaign: 10,000\n- Expected Revenue in Campaign: 100,000 (10x the budget)\n- Click “Save”\n\n3) Invite leads to the campaign\n- Click “Add Leads”\n- Select checkboxes for “Ali Alaba” and “Leo Messi”\n- Click “Next”\n- Click “Submit”\n\nNotes:\n- No hotkeys were used.\nSample 2:\nTASK:\nCreate a new campaign with following details. [name: Autumn Trail Running Launch; type: Direct Mail; Start Date: four weeks from today; Budget: 10,000; Expected Revenue in Campaign: 3 times the Budget.] Invite the following lead to this campaign: [Amanda Klein]\nEXPERIENCE:\nSuccessful plan to create the campaign and invite the lead:\n\n- Log in to the app.\n- Navigate to Campaigns and click New.\n- Enter campaign details:\n - Campaign Name: Autumn Trail Running Launch\n - Type: open the Type dropdown, scroll, and select Direct Mail\n - Start Date: 09/15/2025 (four weeks from the date used in the trajectory)\n - Expected Revenue in Campaign: 30,000\n - Budgeted Cost in Campaign: 10,000\n- Click Save.\n- Add the lead to the campaign:\n - Click Add Leads\n - Select the checkbox for Amanda Klein\n - Click Next, then Submit\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/229"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_sales.py#L491","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_sales.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_campaign","data_retrieval_spec":[{"type":"soql","id":"campaign_info","params":{"query":"SELECT Id, Type, StartDate, BudgetedCost, ExpectedRevenue FROM Campaign WHERE Name='{campaign_name}'"}},{"type":"soql","id":"campaign_members","params":{"query":"SELECT Lead.Name FROM CampaignMember WHERE Campaign.Name='{campaign_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0229fbf7bad5ec66cd39589a3f54733caf2040b3d145ac596f9d719adffd91c7"} {"id":"scuba:sales_013_001","source_id":"scuba","original_id":"sales_013_001","kind":"benchmark_task","instruction":"From the most recent VoiceCallTranscript record, can we qualify Ali Hussein as a lead? If not, reply with only one word—'Budget', 'Authority', 'Need', or 'Timeline'—to indicate the blocker. If yes, reply with \"Qualified\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_001","query_templated":"Can the lead Ali Hussein be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"From the most recent VoiceCallTranscript record, can we qualify Ali Hussein as a lead? If not, reply with only one word—'Budget', 'Authority', 'Need', or 'Timeline'—to indicate the blocker. If yes, reply with \"Qualified\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Authority"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Ali Hussein"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/230","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_013","template_name":"qa_sales_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/230"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/230"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_001","original":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the most recent VoiceCallTranscript record, can we qualify Ali Hussein as a lead? If not, reply with only one word—'Budget', 'Authority', 'Need', or 'Timeline'—to indicate the blocker. If yes, reply with \"Qualified\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_001","query_templated":"Can the lead Ali Hussein be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the most recent VoiceCallTranscript record, can we qualify Ali Hussein as a lead? If not, reply with only one word—'Budget', 'Authority', 'Need', or 'Timeline'—to indicate the blocker. If yes, reply with \"Qualified\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Authority"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Ali Hussein"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/230"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"92292da0bb78b94d5341611cc8dd4c4611aa8d0d4296f4817297586f0af82e04"} {"id":"scuba:sales_013_002","source_id":"scuba","original_id":"sales_013_002","kind":"benchmark_task","instruction":"Looking at our latest VoiceCallTranscript record, would you green-light Raj Patel? If the answer is no, what single issue is blocking it—'Budget', 'Authority', 'Need', or 'Timeline'? Respond with just that one word. If you would green-light, respond with \"Qualified\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_002","query_templated":"Can the lead Raj Patel be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"Looking at our latest VoiceCallTranscript record, would you green-light Raj Patel? If the answer is no, what single issue is blocking it—'Budget', 'Authority', 'Need', or 'Timeline'? Respond with just that one word. If you would green-light, respond with \"Qualified\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Budget"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Raj Patel"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/231","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_013","template_name":"qa_sales_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/231"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/231"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_001","original":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nLooking at our latest VoiceCallTranscript record, would you green-light Raj Patel? If the answer is no, what single issue is blocking it—'Budget', 'Authority', 'Need', or 'Timeline'? Respond with just that one word. If you would green-light, respond with \"Qualified\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_002","query_templated":"Can the lead Raj Patel be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nLooking at our latest VoiceCallTranscript record, would you green-light Raj Patel? If the answer is no, what single issue is blocking it—'Budget', 'Authority', 'Need', or 'Timeline'? Respond with just that one word. If you would green-light, respond with \"Qualified\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Budget"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Raj Patel"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/231"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"e63e5e00650c7906724f3f71123e30e020d07151371c5538a2fe45d82a13e8b2"} {"id":"scuba:sales_013_003","source_id":"scuba","original_id":"sales_013_003","kind":"benchmark_task","instruction":"For Wei Chen, check the newest VoiceCallTranscript record and give a single-word outcome. If they qualify, return \"Qualified\"; if not, return only the blocking factor: 'Budget', 'Authority', 'Need', or 'Timeline'.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_003","query_templated":"Can the lead Wei Chen be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"For Wei Chen, check the newest VoiceCallTranscript record and give a single-word outcome. If they qualify, return \"Qualified\"; if not, return only the blocking factor: 'Budget', 'Authority', 'Need', or 'Timeline'.","human_review_comment":"accept","ground_truth_dict":{"answer":"Qualified"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Wei Chen"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/232","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_013","template_name":"qa_sales_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/232"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/232"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_001","original":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFor Wei Chen, check the newest VoiceCallTranscript record and give a single-word outcome. If they qualify, return \"Qualified\"; if not, return only the blocking factor: 'Budget', 'Authority', 'Need', or 'Timeline'.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_003","query_templated":"Can the lead Wei Chen be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFor Wei Chen, check the newest VoiceCallTranscript record and give a single-word outcome. If they qualify, return \"Qualified\"; if not, return only the blocking factor: 'Budget', 'Authority', 'Need', or 'Timeline'.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Qualified"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Wei Chen"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/232"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"b17f51360b9e15e7390f1b1afcfb729475c11fd44339342e355833442b4ba526"} {"id":"scuba:sales_013_004","source_id":"scuba","original_id":"sales_013_004","kind":"benchmark_task","instruction":"Run a BANT-style check on Liam Chen using the latest VoiceCallTranscript record. If some criterion prevents qualification, output only that criterion ('Budget', 'Authority', 'Need', or 'Timeline'). If none do, output \"Qualified\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_004","query_templated":"Can the lead Liam Chen be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"Run a BANT-style check on Liam Chen using the latest VoiceCallTranscript record. If some criterion prevents qualification, output only that criterion ('Budget', 'Authority', 'Need', or 'Timeline'). If none do, output \"Qualified\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Timeline"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Liam Chen"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/233","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_013","template_name":"qa_sales_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/233"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/233"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_001","original":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nRun a BANT-style check on Liam Chen using the latest VoiceCallTranscript record. If some criterion prevents qualification, output only that criterion ('Budget', 'Authority', 'Need', or 'Timeline'). If none do, output \"Qualified\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_004","query_templated":"Can the lead Liam Chen be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nRun a BANT-style check on Liam Chen using the latest VoiceCallTranscript record. If some criterion prevents qualification, output only that criterion ('Budget', 'Authority', 'Need', or 'Timeline'). If none do, output \"Qualified\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Timeline"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Liam Chen"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/233"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"2fdcec093085d3f2fbd2c8529f21c3613379d693e07e8612a0273f0cf4dc3b50"} {"id":"scuba:sales_013_005","source_id":"scuba","original_id":"sales_013_005","kind":"benchmark_task","instruction":"After reviewing the latest VoiceCallTranscript record, decide whether Fatima Al-Mansouri should be qualified. If she isn’t, name just one culprit—'Budget', 'Authority', 'Need', or 'Timeline'—and return only that word. If she is, return \"Qualified\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_005","query_templated":"Can the lead Fatima Al-Mansouri be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"After reviewing the latest VoiceCallTranscript record, decide whether Fatima Al-Mansouri should be qualified. If she isn’t, name just one culprit—'Budget', 'Authority', 'Need', or 'Timeline'—and return only that word. If she is, return \"Qualified\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Need"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Fatima Al-Mansouri"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/234","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_013","template_name":"qa_sales_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/234"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/234"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_001","original":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_001","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nAfter reviewing the latest VoiceCallTranscript record, decide whether Fatima Al-Mansouri should be qualified. If she isn’t, name just one culprit—'Budget', 'Authority', 'Need', or 'Timeline'—and return only that word. If she is, return \"Qualified\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_013_005","query_templated":"Can the lead Fatima Al-Mansouri be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nAfter reviewing the latest VoiceCallTranscript record, decide whether Fatima Al-Mansouri should be qualified. If she isn’t, name just one culprit—'Budget', 'Authority', 'Need', or 'Timeline'—and return only that word. If she is, return \"Qualified\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Need"},"difficulty":"hard","query_template_name":"qa_sales_001","query_template_metadata":{"template_id":"sales_013","template_string":"Can the lead {lead_name} be qualified based on the latest discussions? If the answer is no, which factor—'Budget', 'Authority', 'Need', or 'Timeline'—is responsible? Return only one factor that the lead qualification fails to meet (i.e. 'Budget', 'Authority', 'Need', 'Timeline'). If the lead is qualified, return \"Qualified\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["LeadCall"]}},"objects":["VoiceCallTranscript__c","Lead"],"metadata_types":["ValidationRule"]},"instance_dict":{"lead_name":"Fatima Al-Mansouri"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/234"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"b0ec97a24c72f714c9b6c50603ab6de6fdbfd0e2812639a2b272b755fab4f03e"} {"id":"scuba:sales_014_001","source_id":"scuba","original_id":"sales_014_001","kind":"benchmark_task","instruction":"Could you sanity-check the pipeline stage on the opportunity \"QuantumLeap Integration Opportunity\"? Does the current stage label truly match what's happening in the deal? If not, pick the correct stage from this list and return only that label: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already right, return \"Correct\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_001","query_templated":"Is the current stage name accurately representing the activity for the opportunity QuantumLeap Integration Opportunity? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"Could you sanity-check the pipeline stage on the opportunity \"QuantumLeap Integration Opportunity\"? Does the current stage label truly match what's happening in the deal? If not, pick the correct stage from this list and return only that label: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already right, return \"Correct\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Proposal/Price Quote"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"QuantumLeap Integration Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/235","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_014","template_name":"qa_sales_002","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/235"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/235"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_002","original":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you sanity-check the pipeline stage on the opportunity \"QuantumLeap Integration Opportunity\"? Does the current stage label truly match what's happening in the deal? If not, pick the correct stage from this list and return only that label: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already right, return \"Correct\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_001","query_templated":"Is the current stage name accurately representing the activity for the opportunity QuantumLeap Integration Opportunity? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nCould you sanity-check the pipeline stage on the opportunity \"QuantumLeap Integration Opportunity\"? Does the current stage label truly match what's happening in the deal? If not, pick the correct stage from this list and return only that label: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already right, return \"Correct\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Proposal/Price Quote"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"QuantumLeap Integration Opportunity"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/235"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"22ebd022df1f2195a776a4c89bb349179ea4cb5558bf241ca175d402cb6ad813"} {"id":"scuba:sales_014_002","source_id":"scuba","original_id":"sales_014_002","kind":"benchmark_task","instruction":"Review the opportunity \"Quantum Dynamics EDA Integration\" and verify whether its stage name reflects the actual activity. If there's a mismatch, output the proper stage chosen strictly from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the current stage is accurate, output \"Correct\". Return only the single label or \"Correct\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_002","query_templated":"Is the current stage name accurately representing the activity for the opportunity Quantum Dynamics EDA Integration? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"Review the opportunity \"Quantum Dynamics EDA Integration\" and verify whether its stage name reflects the actual activity. If there's a mismatch, output the proper stage chosen strictly from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the current stage is accurate, output \"Correct\". Return only the single label or \"Correct\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Negotiation/Review"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Quantum Dynamics EDA Integration"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/236","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_014","template_name":"qa_sales_002","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/236"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/236"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_002","original":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nReview the opportunity \"Quantum Dynamics EDA Integration\" and verify whether its stage name reflects the actual activity. If there's a mismatch, output the proper stage chosen strictly from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the current stage is accurate, output \"Correct\". Return only the single label or \"Correct\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_002","query_templated":"Is the current stage name accurately representing the activity for the opportunity Quantum Dynamics EDA Integration? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nReview the opportunity \"Quantum Dynamics EDA Integration\" and verify whether its stage name reflects the actual activity. If there's a mismatch, output the proper stage chosen strictly from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the current stage is accurate, output \"Correct\". Return only the single label or \"Correct\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Negotiation/Review"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Quantum Dynamics EDA Integration"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/236"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"1b1d5a8a9df5f00479555138fd92894d81a006f44aa1aa6324d6be98980de7b1"} {"id":"scuba:sales_014_003","source_id":"scuba","original_id":"sales_014_003","kind":"benchmark_task","instruction":"I'm validating our opportunity for \"Innovative EDA Partnership with FutureTech\". Is the current stage name aligned with the work underway? If you'd move it, which stage would you switch it to from this set: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost')? Reply with only the chosen stage label, or \"Correct\" if no change is needed.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_003","query_templated":"Is the current stage name accurately representing the activity for the opportunity Innovative EDA Partnership with FutureTech? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"I'm validating our opportunity for \"Innovative EDA Partnership with FutureTech\". Is the current stage name aligned with the work underway? If you'd move it, which stage would you switch it to from this set: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost')? Reply with only the chosen stage label, or \"Correct\" if no change is needed.","human_review_comment":"accept","ground_truth_dict":{"answer":"Needs Analysis"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Innovative EDA Partnership with FutureTech"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/237","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_014","template_name":"qa_sales_002","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/237"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/237"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_002","original":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nI'm validating our opportunity for \"Innovative EDA Partnership with FutureTech\". Is the current stage name aligned with the work underway? If you'd move it, which stage would you switch it to from this set: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost')? Reply with only the chosen stage label, or \"Correct\" if no change is needed.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_003","query_templated":"Is the current stage name accurately representing the activity for the opportunity Innovative EDA Partnership with FutureTech? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nI'm validating our opportunity for \"Innovative EDA Partnership with FutureTech\". Is the current stage name aligned with the work underway? If you'd move it, which stage would you switch it to from this set: ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost')? Reply with only the chosen stage label, or \"Correct\" if no change is needed.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Needs Analysis"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"Innovative EDA Partnership with FutureTech"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/237"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"605854c9a4ab198a0869cb3aa74a5e747dab6f7e1d5203873f9d62782663dd26"} {"id":"scuba:sales_014_004","source_id":"scuba","original_id":"sales_014_004","kind":"benchmark_task","instruction":"Please audit the stage for the opportunity \"EcoEnergy Solutions - Advanced EDA Integration\" to ensure it matches the deal's activity. When it's wrong, map it to the appropriate stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already correct, return \"Correct\". Only return the single stage label or \"Correct\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_004","query_templated":"Is the current stage name accurately representing the activity for the opportunity EcoEnergy Solutions - Advanced EDA Integration? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"Please audit the stage for the opportunity \"EcoEnergy Solutions - Advanced EDA Integration\" to ensure it matches the deal's activity. When it's wrong, map it to the appropriate stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already correct, return \"Correct\". Only return the single stage label or \"Correct\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Closed Won"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"EcoEnergy Solutions - Advanced EDA Integration"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/238","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_014","template_name":"qa_sales_002","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/238"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/238"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_002","original":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease audit the stage for the opportunity \"EcoEnergy Solutions - Advanced EDA Integration\" to ensure it matches the deal's activity. When it's wrong, map it to the appropriate stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already correct, return \"Correct\". Only return the single stage label or \"Correct\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_004","query_templated":"Is the current stage name accurately representing the activity for the opportunity EcoEnergy Solutions - Advanced EDA Integration? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease audit the stage for the opportunity \"EcoEnergy Solutions - Advanced EDA Integration\" to ensure it matches the deal's activity. When it's wrong, map it to the appropriate stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If it's already correct, return \"Correct\". Only return the single stage label or \"Correct\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Closed Won"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"EcoEnergy Solutions - Advanced EDA Integration"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/238"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"f4ba0d2b12523aa3bdae78836b5aac2700b5310a718c937857861db11c9c60da"} {"id":"scuba:sales_014_005","source_id":"scuba","original_id":"sales_014_005","kind":"benchmark_task","instruction":"Quick gut check on the opportunity \"DigitalWave-TechPulse Strategic Partnership\": is the current stage actually the right fit? If not, select the best-matching stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). Respond with just that label; if the existing stage is fine, return \"Correct\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_005","query_templated":"Is the current stage name accurately representing the activity for the opportunity DigitalWave-TechPulse Strategic Partnership? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"Quick gut check on the opportunity \"DigitalWave-TechPulse Strategic Partnership\": is the current stage actually the right fit? If not, select the best-matching stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). Respond with just that label; if the existing stage is fine, return \"Correct\".","human_review_comment":"accept","ground_truth_dict":{"answer":"Correct"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"DigitalWave-TechPulse Strategic Partnership"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/239","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"hard","template_id":"sales_014","template_name":"qa_sales_002","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/239"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/239"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_002","original":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_002","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nQuick gut check on the opportunity \"DigitalWave-TechPulse Strategic Partnership\": is the current stage actually the right fit? If not, select the best-matching stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). Respond with just that label; if the existing stage is fine, return \"Correct\".\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_014_005","query_templated":"Is the current stage name accurately representing the activity for the opportunity DigitalWave-TechPulse Strategic Partnership? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nQuick gut check on the opportunity \"DigitalWave-TechPulse Strategic Partnership\": is the current stage actually the right fit? If not, select the best-matching stage from ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). Respond with just that label; if the existing stage is fine, return \"Correct\".\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Correct"},"difficulty":"hard","query_template_name":"qa_sales_002","query_template_metadata":{"template_id":"sales_014","template_string":"Is the current stage name accurately representing the activity for the opportunity {opportunity_name}? If it isn't, what would be the appropriate stage name? Return only the correct stage label among ('Prospecting', 'Qualification', 'Needs Analysis', 'Value Proposition', 'Id. Decision Makers', 'Perception Analysis', 'Proposal/Price Quote', 'Negotiation/Review', 'Closed Won', 'Closed Lost'). If the stage name is correct, return \"Correct\".","prerequisites":{"data":{"objects":["OppTask"]}},"objects":["Task","Opportunity"],"metadata_types":["ValidationRule"]},"instance_dict":{"opportunity_name":"DigitalWave-TechPulse Strategic Partnership"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/239"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"39b2b057331d1fdc2a8cf4dfe37679f083e9a1220287bf83abe79e7450b3faa3"} {"id":"scuba:sales_015_001","source_id":"scuba","original_id":"sales_015_001","kind":"benchmark_task","instruction":"Quick favor—scan the VoiceCallTranscript record between Carlos Nunez and Hiro Tanaka and tell me how many years the warranty lasts on the product Carlos offers Hiro. Reply with the number only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_001","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Carlos Nunez and Hiro Tanaka, and determine how many years the warranty lasts for the product that Carlos offers Hiro. Respond with the number only.","query":"Quick favor—scan the VoiceCallTranscript record between Carlos Nunez and Hiro Tanaka and tell me how many years the warranty lasts on the product Carlos offers Hiro. Reply with the number only.","human_review_comment":"accept","ground_truth_dict":{"answer":"2"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Carlos Nunez","caller2":"Hiro Tanaka","action":"determine how many years the warranty lasts for the product that Carlos offers Hiro. Respond with the number only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/240","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"easy","template_id":"sales_015","template_name":"qa_sales_003","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/240"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/240"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_003","original":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nQuick favor—scan the VoiceCallTranscript record between Carlos Nunez and Hiro Tanaka and tell me how many years the warranty lasts on the product Carlos offers Hiro. Reply with the number only.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_001","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Carlos Nunez and Hiro Tanaka, and determine how many years the warranty lasts for the product that Carlos offers Hiro. Respond with the number only.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nQuick favor—scan the VoiceCallTranscript record between Carlos Nunez and Hiro Tanaka and tell me how many years the warranty lasts on the product Carlos offers Hiro. Reply with the number only.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"2"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Carlos Nunez","caller2":"Hiro Tanaka","action":"determine how many years the warranty lasts for the product that Carlos offers Hiro. Respond with the number only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/240"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"7d8fb6c2cd06f255d3210dc0afcbca2cdc6c5609db2c90bf646a9d090056bf27"} {"id":"scuba:sales_015_002","source_id":"scuba","original_id":"sales_015_002","kind":"benchmark_task","instruction":"In the VoiceCallTranscript record for Carlos Fernandez and Rajesh Kumar, what price does Carlos quote Rajesh for one unit of AutoLayout Master? Respond with the number plus the currency symbol only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_002","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Carlos Fernandez and Rajesh Kumar, and find out what is the price that Carlos offers Rajesh for one unit of the AutoLayout Master. Respond with the number with the currency symbol only.","query":"In the VoiceCallTranscript record for Carlos Fernandez and Rajesh Kumar, what price does Carlos quote Rajesh for one unit of AutoLayout Master? Respond with the number plus the currency symbol only.","human_review_comment":"accept","ground_truth_dict":{"answer":"$530"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Carlos Fernandez","caller2":"Rajesh Kumar","action":"find out what is the price that Carlos offers Rajesh for one unit of the AutoLayout Master. Respond with the number with the currency symbol only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/241","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"easy","template_id":"sales_015","template_name":"qa_sales_003","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/241"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/241"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_003","original":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the VoiceCallTranscript record for Carlos Fernandez and Rajesh Kumar, what price does Carlos quote Rajesh for one unit of AutoLayout Master? Respond with the number plus the currency symbol only.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_002","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Carlos Fernandez and Rajesh Kumar, and find out what is the price that Carlos offers Rajesh for one unit of the AutoLayout Master. Respond with the number with the currency symbol only.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nIn the VoiceCallTranscript record for Carlos Fernandez and Rajesh Kumar, what price does Carlos quote Rajesh for one unit of AutoLayout Master? Respond with the number plus the currency symbol only.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"$530"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Carlos Fernandez","caller2":"Rajesh Kumar","action":"find out what is the price that Carlos offers Rajesh for one unit of the AutoLayout Master. Respond with the number with the currency symbol only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/241"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"da7be9d47e8b7da456b3e5858f9730153731e3a9109258fcb84a6fa119c3ba30"} {"id":"scuba:sales_015_003","source_id":"scuba","original_id":"sales_015_003","kind":"benchmark_task","instruction":"From the VoiceCallTranscript record between Emmanuel Okonkwo and Rajesh Kumar, within how many days does Rajesh want the installation completed? Answer with the number only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_003","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Emmanuel Okonkwo and Rajesh Kumar, and tell me within how many days does the customer Rajesh wish to have the installation done. Respond with the number only.","query":"From the VoiceCallTranscript record between Emmanuel Okonkwo and Rajesh Kumar, within how many days does Rajesh want the installation completed? Answer with the number only.","human_review_comment":"accept","ground_truth_dict":{"answer":"3"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Emmanuel Okonkwo","caller2":"Rajesh Kumar","action":"tell me within how many days does the customer Rajesh wish to have the installation done. Respond with the number only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/242","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"easy","template_id":"sales_015","template_name":"qa_sales_003","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/242"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/242"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_003","original":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the VoiceCallTranscript record between Emmanuel Okonkwo and Rajesh Kumar, within how many days does Rajesh want the installation completed? Answer with the number only.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_003","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Emmanuel Okonkwo and Rajesh Kumar, and tell me within how many days does the customer Rajesh wish to have the installation done. Respond with the number only.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nFrom the VoiceCallTranscript record between Emmanuel Okonkwo and Rajesh Kumar, within how many days does Rajesh want the installation completed? Answer with the number only.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"3"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Emmanuel Okonkwo","caller2":"Rajesh Kumar","action":"tell me within how many days does the customer Rajesh wish to have the installation done. Respond with the number only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/242"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"bf0cb8ceb5822bdf58b5ce6fcea5ad03c6718f84e00df4e1e9401e613a1e09df"} {"id":"scuba:sales_015_004","source_id":"scuba","original_id":"sales_015_004","kind":"benchmark_task","instruction":"Please review the VoiceCallTranscript record between Fatima Al-Masri and Xia Li and identify Xia’s maximum budget for the purchase. Return the amount only, including the currency symbol.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_004","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Fatima Al-Masri and Xia Li, and tell me what is the limit of Xia's budget for the purchase. Respond with the number with the currency symbol only.","query":"Please review the VoiceCallTranscript record between Fatima Al-Masri and Xia Li and identify Xia’s maximum budget for the purchase. Return the amount only, including the currency symbol.","human_review_comment":"accept","ground_truth_dict":{"answer":"$2,155"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Fatima Al-Masri","caller2":"Xia Li","action":"tell me what is the limit of Xia's budget for the purchase. Respond with the number with the currency symbol only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/243","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"easy","template_id":"sales_015","template_name":"qa_sales_003","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/243"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/243"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_003","original":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease review the VoiceCallTranscript record between Fatima Al-Masri and Xia Li and identify Xia’s maximum budget for the purchase. Return the amount only, including the currency symbol.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_004","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Fatima Al-Masri and Xia Li, and tell me what is the limit of Xia's budget for the purchase. Respond with the number with the currency symbol only.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nPlease review the VoiceCallTranscript record between Fatima Al-Masri and Xia Li and identify Xia’s maximum budget for the purchase. Return the amount only, including the currency symbol.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"$2,155"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Fatima Al-Masri","caller2":"Xia Li","action":"tell me what is the limit of Xia's budget for the purchase. Respond with the number with the currency symbol only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/243"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"bb7aaed5e780e1ef359afb6be2468c842d2ff29b5f790082882416a21f250cfc"} {"id":"scuba:sales_015_005","source_id":"scuba","original_id":"sales_015_005","kind":"benchmark_task","instruction":"Looking at the VoiceCallTranscript record between Monique Dubois and Olga Sokolova, which product does Monique propose to address testing and optimization of configurations in a virtual environment? Provide the precise product name only.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_005","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Monique Dubois and Olga Sokolova, and tell me which product Monique is offering to Olga as a solution to the need of testing and optimizing the configurations in a virtual environment. Respond with the product name only.","query":"Looking at the VoiceCallTranscript record between Monique Dubois and Olga Sokolova, which product does Monique propose to address testing and optimization of configurations in a virtual environment? Provide the precise product name only.","human_review_comment":"accept","ground_truth_dict":{"answer":"UnitySim"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Monique Dubois","caller2":"Olga Sokolova","action":"tell me which product Monique is offering to Olga as a solution to the need of testing and optimizing the configurations in a virtual environment. Respond with the product name only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/244","source_record_type":"crm_workflow_instance","category":"sales","role":"sales","subcategory":"qa","source_difficulty":"easy","template_id":"sales_015","template_name":"qa_sales_003","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/244"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/244"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003"}],"source_schema_notes":[],"template_registry":{"name":"qa_sales_003","original":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_sales_003","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nLooking at the VoiceCallTranscript record between Monique Dubois and Olga Sokolova, which product does Monique propose to address testing and optimization of configurations in a virtual environment? Provide the precise product name only.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"sales","subcategory":"qa","task_id":"sales_015_005","query_templated":"I'm reviewing the call transcripts to extract useful information. Please examine the call between Monique Dubois and Olga Sokolova, and tell me which product Monique is offering to Olga as a solution to the need of testing and optimizing the configurations in a virtual environment. Respond with the product name only.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nLooking at the VoiceCallTranscript record between Monique Dubois and Olga Sokolova, which product does Monique propose to address testing and optimization of configurations in a virtual environment? Provide the precise product name only.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"UnitySim"},"difficulty":"easy","query_template_name":"qa_sales_003","query_template_metadata":{"template_id":"sales_015","template_string":"I'm reviewing the call transcripts to extract useful information. Please examine the call between {caller1} and {caller2}, and {action}.","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["VoiceCallTranscript__c","Knowledge__kav"],"CustomTab":["VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"],"Settings":["Knowledge"]}},"data":{"objects":["VoiceCallTranscript__c"]}},"objects":["VoiceCallTranscript__c"],"metadata_types":[]},"instance_dict":{"caller1":"Monique Dubois","caller2":"Olga Sokolova","action":"tell me which product Monique is offering to Olga as a solution to the need of testing and optimizing the configurations in a virtual environment. Respond with the product name only"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/244"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"64db4f5e849fe9000987f58186203a7533f404dd2400f2370604e3f1853f936c"} {"id":"scuba:service_001_001","source_id":"scuba","original_id":"service_001_001","kind":"benchmark_task","instruction":"Create a queue named \"Shoe Case Support.\" Limit notification emails so they go only to the \"support.shoe@papaltd.com\" distribution list. Add the Customer object, and add the role \"Customer Support, North America\" as a member.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_001","query_templated":"Create a queue with name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.","query":"Create a queue named \"Shoe Case Support.\" Limit notification emails so they go only to the \"support.shoe@papaltd.com\" distribution list. Add the Customer object, and add the role \"Customer Support, North America\" as a member.","human_review_comment":"tweak a bit","ground_truth_dict":{"queue_name":"shoe_case_support","group_type":"Customer","email_address":"support.shoe@papaltd.com","role_members":["Customer Support, North America"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"Shoe Case Support","email_instruction":"Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\"","role_users":"Role: Customer Support, North America"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/245","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_001","template_name":"create_queue_with_members","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/245"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/245"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_queue_with_members"}],"source_schema_notes":[],"template_registry":{"name":"create_queue_with_members","original":{"template_id":"service_001","template_string":"Create a queue for case management with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["users.json"]}}},"objects":["Queue","QueueSobject"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members","differs_from_embedded_fields":["metadata_types","objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a queue named \"Shoe Case Support.\" Limit notification emails so they go only to the \"support.shoe@papaltd.com\" distribution list. Add the Customer object, and add the role \"Customer Support, North America\" as a member.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_001","query_templated":"Create a queue with name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a queue named \"Shoe Case Support.\" Limit notification emails so they go only to the \"support.shoe@papaltd.com\" distribution list. Add the Customer object, and add the role \"Customer Support, North America\" as a member.\n```","human_review_comment":"tweak a bit","ground_truth_dict":{"queue_name":"shoe_case_support","group_type":"Customer","email_address":"support.shoe@papaltd.com","role_members":["Customer Support, North America"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"Shoe Case Support","email_instruction":"Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\"","role_users":"Role: Customer Support, North America"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/245"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L38","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_queue_with_members","data_retrieval_spec":[{"type":"soql","id":"queue_check","params":{"query":"SELECT Id, DoesSendEmailToMembers, Email FROM Group WHERE Type = 'Queue' AND DeveloperName = '{queue_name}'"}},{"type":"soql","id":"object_type_check","params":{"query":"SELECT QueueId, Queue.Name, SobjectType FROM QueueSobject WHERE QueueId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"member_check","params":{"query":"SELECT UserOrGroup.Name, UserOrGroup.Email FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"role_related_id","params":{"query":"SELECT Id, Name, Type, RelatedId FROM Group WHERE Id IN (SELECT UserOrGroupId FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}')"}},{"type":"soql","id":"role_check","params":{"query":"SELECT Id, Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c4175abcc25f0def16abd21521aa1af7b6aad334bcf77a632d438cc39951bf32"} {"id":"scuba:service_001_002","source_id":"scuba","original_id":"service_001_002","kind":"benchmark_task","instruction":"Create a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_002","query_templated":"Create a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.","query":"Create a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.","human_review_comment":"reject; use the original","ground_truth_dict":{"queue_name":"vip_support","group_type":"BuyerGroup","email_address":"","role_members":["SVP, Customer Service & Support","Customer Support, International"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"VIP Support","email_instruction":"Add the Object \"Buyer Group\"","role_users":"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/246","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_001","template_name":"create_queue_with_members","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/246"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/246"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_queue_with_members"}],"source_schema_notes":[],"template_registry":{"name":"create_queue_with_members","original":{"template_id":"service_001","template_string":"Create a queue for case management with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["users.json"]}}},"objects":["Queue","QueueSobject"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members","differs_from_embedded_fields":["metadata_types","objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_002","query_templated":"Create a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCreate a queue with name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\n```","human_review_comment":"reject; use the original","ground_truth_dict":{"queue_name":"vip_support","group_type":"BuyerGroup","email_address":"","role_members":["SVP, Customer Service & Support","Customer Support, International"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"VIP Support","email_instruction":"Add the Object \"Buyer Group\"","role_users":"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/246"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L38","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_queue_with_members","data_retrieval_spec":[{"type":"soql","id":"queue_check","params":{"query":"SELECT Id, DoesSendEmailToMembers, Email FROM Group WHERE Type = 'Queue' AND DeveloperName = '{queue_name}'"}},{"type":"soql","id":"object_type_check","params":{"query":"SELECT QueueId, Queue.Name, SobjectType FROM QueueSobject WHERE QueueId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"member_check","params":{"query":"SELECT UserOrGroup.Name, UserOrGroup.Email FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"role_related_id","params":{"query":"SELECT Id, Name, Type, RelatedId FROM Group WHERE Id IN (SELECT UserOrGroupId FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}')"}},{"type":"soql","id":"role_check","params":{"query":"SELECT Id, Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"05ad79cbe401fffa1f29e6cf61ec445d68dfd080108b765091592ea2f7cddf24"} {"id":"scuba:service_001_003","source_id":"scuba","original_id":"service_001_003","kind":"benchmark_task","instruction":"Please create a queue titled \"Marketing Support,\" and configure email alerts to be sent exclusively to the \"marketing.support@business.com\" distribution list. Link it to the Attribute Picklist object, and add the \"User: Integration User\" plus the role \"Role: Marketing Team\" as members.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_003","query_templated":"Create a queue with name \"Marketing Support\". Only send emails to members under the distribution list \"marketing.support@business.com\". Add the Object \"Attribute Picklist\". Include \"User: Integration User\" and \"Role: Marketing Team\" as the Members.","query":"Please create a queue titled \"Marketing Support,\" and configure email alerts to be sent exclusively to the \"marketing.support@business.com\" distribution list. Link it to the Attribute Picklist object, and add the \"User: Integration User\" plus the role \"Role: Marketing Team\" as members.","human_review_comment":"tweak a bit","ground_truth_dict":{"queue_name":"Marketing_Support","group_type":"AttributePicklist","email_address":"marketing.support@business.com","role_members":["Integration User","Marketing Team"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"Marketing Support","email_instruction":"Only send emails to members under the distribution list \"marketing.support@business.com\". Add the Object \"Attribute Picklist\"","role_users":"User: Integration User\" and \"Role: Marketing Team"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/247","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_001","template_name":"create_queue_with_members","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/247"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/247"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_queue_with_members"}],"source_schema_notes":[],"template_registry":{"name":"create_queue_with_members","original":{"template_id":"service_001","template_string":"Create a queue for case management with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["users.json"]}}},"objects":["Queue","QueueSobject"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members","differs_from_embedded_fields":["metadata_types","objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease create a queue titled \"Marketing Support,\" and configure email alerts to be sent exclusively to the \"marketing.support@business.com\" distribution list. Link it to the Attribute Picklist object, and add the \"User: Integration User\" plus the role \"Role: Marketing Team\" as members.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_003","query_templated":"Create a queue with name \"Marketing Support\". Only send emails to members under the distribution list \"marketing.support@business.com\". Add the Object \"Attribute Picklist\". Include \"User: Integration User\" and \"Role: Marketing Team\" as the Members.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease create a queue titled \"Marketing Support,\" and configure email alerts to be sent exclusively to the \"marketing.support@business.com\" distribution list. Link it to the Attribute Picklist object, and add the \"User: Integration User\" plus the role \"Role: Marketing Team\" as members.\n```","human_review_comment":"tweak a bit","ground_truth_dict":{"queue_name":"Marketing_Support","group_type":"AttributePicklist","email_address":"marketing.support@business.com","role_members":["Integration User","Marketing Team"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"Marketing Support","email_instruction":"Only send emails to members under the distribution list \"marketing.support@business.com\". Add the Object \"Attribute Picklist\"","role_users":"User: Integration User\" and \"Role: Marketing Team"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/247"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L38","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_queue_with_members","data_retrieval_spec":[{"type":"soql","id":"queue_check","params":{"query":"SELECT Id, DoesSendEmailToMembers, Email FROM Group WHERE Type = 'Queue' AND DeveloperName = '{queue_name}'"}},{"type":"soql","id":"object_type_check","params":{"query":"SELECT QueueId, Queue.Name, SobjectType FROM QueueSobject WHERE QueueId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"member_check","params":{"query":"SELECT UserOrGroup.Name, UserOrGroup.Email FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"role_related_id","params":{"query":"SELECT Id, Name, Type, RelatedId FROM Group WHERE Id IN (SELECT UserOrGroupId FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}')"}},{"type":"soql","id":"role_check","params":{"query":"SELECT Id, Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"52d7f1b90bcdbb2c7903f1f5456b127798a05acc0335b486d7f2ba0408c55c85"} {"id":"scuba:service_001_004","source_id":"scuba","original_id":"service_001_004","kind":"benchmark_task","instruction":"I need a queue named \"tech support specialists\": send notification emails to members of the tech.support@innovatetech.com distribution list, attach the Case object, and add Alice Bob as a member.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_004","query_templated":"Create a queue with name tech support specialists. Send email notifications to members under the distribution list tech.support@innovatetech.com. Include Object \"Case\". Include Alice Bob as the member.","query":"I need a queue named \"tech support specialists\": send notification emails to members of the tech.support@innovatetech.com distribution list, attach the Case object, and add Alice Bob as a member.","human_review_comment":"accept","ground_truth_dict":{"queue_name":"tech_support_specialists","group_type":"Case","email_address":"tech.support@innovatetech.com","role_members":["Alice Bob"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"tech support specialists","email_instruction":"Send email notifications to members under the distribution list tech.support@innovatetech.com. Include Object \"Case\".","role_users":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/248","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_001","template_name":"create_queue_with_members","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/248"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/248"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_queue_with_members"}],"source_schema_notes":[],"template_registry":{"name":"create_queue_with_members","original":{"template_id":"service_001","template_string":"Create a queue for case management with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["users.json"]}}},"objects":["Queue","QueueSobject"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members","differs_from_embedded_fields":["metadata_types","objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI need a queue named \"tech support specialists\": send notification emails to members of the tech.support@innovatetech.com distribution list, attach the Case object, and add Alice Bob as a member.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_004","query_templated":"Create a queue with name tech support specialists. Send email notifications to members under the distribution list tech.support@innovatetech.com. Include Object \"Case\". Include Alice Bob as the member.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI need a queue named \"tech support specialists\": send notification emails to members of the tech.support@innovatetech.com distribution list, attach the Case object, and add Alice Bob as a member.\n```","human_review_comment":"accept","ground_truth_dict":{"queue_name":"tech_support_specialists","group_type":"Case","email_address":"tech.support@innovatetech.com","role_members":["Alice Bob"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"tech support specialists","email_instruction":"Send email notifications to members under the distribution list tech.support@innovatetech.com. Include Object \"Case\".","role_users":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/248"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L38","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_queue_with_members","data_retrieval_spec":[{"type":"soql","id":"queue_check","params":{"query":"SELECT Id, DoesSendEmailToMembers, Email FROM Group WHERE Type = 'Queue' AND DeveloperName = '{queue_name}'"}},{"type":"soql","id":"object_type_check","params":{"query":"SELECT QueueId, Queue.Name, SobjectType FROM QueueSobject WHERE QueueId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"member_check","params":{"query":"SELECT UserOrGroup.Name, UserOrGroup.Email FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"role_related_id","params":{"query":"SELECT Id, Name, Type, RelatedId FROM Group WHERE Id IN (SELECT UserOrGroupId FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}')"}},{"type":"soql","id":"role_check","params":{"query":"SELECT Id, Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f8bfec964706218f03e527d1f0b42cf4a23f9bc158dc172dc99af9c5fa9fb4bb"} {"id":"scuba:service_001_005","source_id":"scuba","original_id":"service_001_005","kind":"benchmark_task","instruction":"Set up a queue called \"enterprise client support,\" associate it with the Case object, and include Alice Bob and Sofia Bennett as members.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_005","query_templated":"Create a queue with name enterprise client support. Include Object \"Case\". Include Alice Bob and Sofia Bennett as the member.","query":"Set up a queue called \"enterprise client support,\" associate it with the Case object, and include Alice Bob and Sofia Bennett as members.","human_review_comment":"accept","ground_truth_dict":{"queue_name":"enterprise_client_support","group_type":"Case","email_address":"","role_members":["Alice Bob","Sofia Bennett"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"enterprise client support","email_instruction":"Include Object \"Case\".","role_users":"Alice Bob and Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/249","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_001","template_name":"create_queue_with_members","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/249"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/249"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_queue_with_members"}],"source_schema_notes":[],"template_registry":{"name":"create_queue_with_members","original":{"template_id":"service_001","template_string":"Create a queue for case management with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["users.json"]}}},"objects":["Queue","QueueSobject"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_queue_with_members","differs_from_embedded_fields":["metadata_types","objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nSet up a queue called \"enterprise client support,\" associate it with the Case object, and include Alice Bob and Sofia Bennett as members.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_001_005","query_templated":"Create a queue with name enterprise client support. Include Object \"Case\". Include Alice Bob and Sofia Bennett as the member.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nSet up a queue called \"enterprise client support,\" associate it with the Case object, and include Alice Bob and Sofia Bennett as members.\n```","human_review_comment":"accept","ground_truth_dict":{"queue_name":"enterprise_client_support","group_type":"Case","email_address":"","role_members":["Alice Bob","Sofia Bennett"]},"difficulty":"easy","query_template_name":"create_queue_with_members","query_template_metadata":{"template_id":"service_001","template_string":"Create a queue with name {queue_name}. {email_instruction}. Include {role_users} as the member.","prerequisites":{"data":{"objects":{"User":["User.json"]}}},"objects":["QueueSobject","Queue"],"metadata_types":["AssignmentRules"]},"instance_dict":{"queue_name":"enterprise client support","email_instruction":"Include Object \"Case\".","role_users":"Alice Bob and Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a Queue with the name \"Shoe Case Support\". Only send email to members under the distribution list \"support.shoe@papaltd.com\". Add the Object \"Customer\". Include \"Role: Customer Support, North America\" as the Member.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in and open Setup.\n- Navigate to Users > Queues.\n- Click New Queue.\n- Configure the queue:\n - Queue Label: Shoe Case Support.\n - Queue Email: support.shoe@papaltd.com.\n- Add supported object:\n - Select Customer and click Add to Selected Objects.\n- Add members:\n - In Queue Members, choose Role, select Role: Customer Support, North America, and click Add to Selected Members.\n- Save the queue.\nSample 2:\nTASK:Create a Queue with the name \"VIP Support\". Add the Object \"Buyer Group\". Include \"Role: SVP, Customer Service & Support\" and \"Role: Customer Support, International\" as the Members.\nEXPERIENCE:\nSuccessful plan to create the Queue:\n\n- Log in and open Setup.\n- In Quick Find, type “Queues” and select Queues.\n- Click “New Queue.”\n- Enter Queue Label: “VIP Support” (Queue Name auto-fills).\n- Scroll to Supported Objects, select “Buyer Group,” and click “Add” to move it to Selected Objects.\n- In Queue Members, set the filter to “Roles.”\n- From Available Members, select “Role: SVP, Customer Service & Support” and click “Add to Selected Members.”\n- Select “Role: Customer Support, International” and click “Add to Selected Members.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/249"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L38","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_queue_with_members","data_retrieval_spec":[{"type":"soql","id":"queue_check","params":{"query":"SELECT Id, DoesSendEmailToMembers, Email FROM Group WHERE Type = 'Queue' AND DeveloperName = '{queue_name}'"}},{"type":"soql","id":"object_type_check","params":{"query":"SELECT QueueId, Queue.Name, SobjectType FROM QueueSobject WHERE QueueId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"member_check","params":{"query":"SELECT UserOrGroup.Name, UserOrGroup.Email FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}'"}},{"type":"soql","id":"role_related_id","params":{"query":"SELECT Id, Name, Type, RelatedId FROM Group WHERE Id IN (SELECT UserOrGroupId FROM GroupMember WHERE GroupId = '{retrieved_results['queue_check'].records[0]['Id']}')"}},{"type":"soql","id":"role_check","params":{"query":"SELECT Id, Name FROM UserRole"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"bd0f6bc8639dee47e806af5bc8b1c1de4190babe3e194ce887261d9858502f57"} {"id":"scuba:service_002_001","source_id":"scuba","original_id":"service_002_001","kind":"benchmark_task","instruction":"Please set up a case assignment rule called Billing Cases Assignment Rule. Route any case to the billing support queue if the description mentions “billing” or if the case number begins with 5. Give the description condition higher precedence, so it’s evaluated first.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_001","query_templated":"Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.","query":"Please set up a case assignment rule called Billing Cases Assignment Rule. Route any case to the billing support queue if the description mentions “billing” or if the case number begins with 5. Give the description condition higher precedence, so it’s evaluated first.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Billing Cases Assignment Rule","entry_conditions":[["Case.Description","contains","billing"],["Case.CaseNumber","startsWith","5"]],"logic_operator":"OR","assignee":"billing_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Billing Cases Assignment Rule","entry_conditions":"cases whose description contains the keyword 'billing' or whose case number starts with 5","assignee":"billing support queue","sort_order_description":"The first condition has higher priority to be evaluated"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/250","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"hard","template_id":"service_002","template_name":"create_case_assignment_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/250"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/250"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_assignment_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_case_assignment_rule","original":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"data":{"objects":{"Group":["groups.json"]}}},"objects":["AssignmentRule"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nPlease set up a case assignment rule called Billing Cases Assignment Rule. Route any case to the billing support queue if the description mentions “billing” or if the case number begins with 5. Give the description condition higher precedence, so it’s evaluated first.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_001","query_templated":"Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.","query":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nPlease set up a case assignment rule called Billing Cases Assignment Rule. Route any case to the billing support queue if the description mentions “billing” or if the case number begins with 5. Give the description condition higher precedence, so it’s evaluated first.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Billing Cases Assignment Rule","entry_conditions":[["Case.Description","contains","billing"],["Case.CaseNumber","startsWith","5"]],"logic_operator":"OR","assignee":"billing_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Billing Cases Assignment Rule","entry_conditions":"cases whose description contains the keyword 'billing' or whose case number starts with 5","assignee":"billing support queue","sort_order_description":"The first condition has higher priority to be evaluated"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/250"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"needs_review","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_assignment_rule","data_retrieval_spec":[{"type":"metadata","id":"assignment_rule_metadata","params":{"types_and_members":{"AssignmentRules":["Case"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path.","review_flags":["condition_precedence_not_checked"],"review_notes":[{"id":"condition_precedence_not_checked","reason_en":"The query requires the description condition to be evaluated first. This evaluator checks condition membership and Boolean logic, but not condition precedence or rule-entry order.","reason_zh":"指令要求优先判断 description 条件；评分器检查条件是否存在及布尔逻辑，不检查条件优先级或规则条目顺序。","evidence":[{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":109,"line_end":127,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":136,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"}]}],"review_provenance":{"review_file":"scuba-verifier-review.json","review_sha256":"1ed7bd039075f446353e7b562e3016aabcfb34c848ace25fa8a3815b4896e836","revision":"b893e2239b035f93d5753a32557346d15935d596","query_sha256":"60a0d7c7fcaedbc49bf4506eb286f411e45afd66853fa92c7d2147337b4cea11","ground_truth_sha256":"cffce925de876f2e434f3029412508a403b77f73f34cbda03254c9d3d4d9b478","hash_encoding":{"query":"UTF-8 bytes of the exact canonical query string","ground_truth":"UTF-8 JSON, sort_keys=true, ensure_ascii=false, separators=(comma,colon)"},"evidence":[{"source_path":"data/test_zero_shot.json","line_start":10460,"line_end":10481,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10460-L10481","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"}],"review_scope":"static_source_review_not_run"}},"instruction_fingerprint":"e746db6d08dc487f987e28a44ca804f5ffb38fd3cf929b1119757f681b2634d0"} {"id":"scuba:service_002_002","source_id":"scuba","original_id":"service_002_002","kind":"benchmark_task","instruction":"Configure a rule named Network Cases Assignment Rule that sends cases to the network support queue only when both are true: the description contains “network” and the case number starts with 7.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_002","query_templated":"Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue. .","query":"Configure a rule named Network Cases Assignment Rule that sends cases to the network support queue only when both are true: the description contains “network” and the case number starts with 7.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Network Cases Assignment Rule","entry_conditions":[["Case.Description","contains","network"],["Case.CaseNumber","startsWith","7"]],"logic_operator":"AND","assignee":"network_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Network Cases Assignment Rule","entry_conditions":"cases whose description contains the keyword 'network' and whose case number starts with 7","assignee":"network support queue","sort_order_description":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/251","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"hard","template_id":"service_002","template_name":"create_case_assignment_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/251"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/251"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_assignment_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_case_assignment_rule","original":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"data":{"objects":{"Group":["groups.json"]}}},"objects":["AssignmentRule"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nConfigure a rule named Network Cases Assignment Rule that sends cases to the network support queue only when both are true: the description contains “network” and the case number starts with 7.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_002","query_templated":"Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue. .","query":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nConfigure a rule named Network Cases Assignment Rule that sends cases to the network support queue only when both are true: the description contains “network” and the case number starts with 7.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Network Cases Assignment Rule","entry_conditions":[["Case.Description","contains","network"],["Case.CaseNumber","startsWith","7"]],"logic_operator":"AND","assignee":"network_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Network Cases Assignment Rule","entry_conditions":"cases whose description contains the keyword 'network' and whose case number starts with 7","assignee":"network support queue","sort_order_description":""},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/251"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_assignment_rule","data_retrieval_spec":[{"type":"metadata","id":"assignment_rule_metadata","params":{"types_and_members":{"AssignmentRules":["Case"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e3c44f6b3a8aba331eb2e43dfea0433a0a263ebdac509b094a27ebd86e68f013"} {"id":"scuba:service_002_003","source_id":"scuba","original_id":"service_002_003","kind":"benchmark_task","instruction":"Create a rule called Shoes. If either condition matches—description includes “shoe” or the case number starts with 2—send the case to the shoe case support queue. Make sure the description check is evaluated first (higher priority).","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_003","query_templated":"Help me with creating a case assignment rule named 'Shoes' by assigning [cases whose description contains the keyword \"shoe\" or whose case number starts with 2] to [shoe case support queue]. [The first condition has higher priority to be evaluated.]","query":"Create a rule called Shoes. If either condition matches—description includes “shoe” or the case number starts with 2—send the case to the shoe case support queue. Make sure the description check is evaluated first (higher priority).","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Shoes","entry_conditions":[["Case.Description","contains","shoe"],["Case.CaseNumber","startsWith","2"]],"logic_operator":"OR","assignee":"shoe_case_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Shoes","entry_conditions":"cases whose description contains the keyword \"shoe\" or whose case number starts with 2","assignee":"shoe case support queue","sort_order_description":"The first condition has higher priority to be evaluated."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/252","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"hard","template_id":"service_002","template_name":"create_case_assignment_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/252"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/252"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_assignment_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_case_assignment_rule","original":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"data":{"objects":{"Group":["groups.json"]}}},"objects":["AssignmentRule"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nCreate a rule called Shoes. If either condition matches—description includes “shoe” or the case number starts with 2—send the case to the shoe case support queue. Make sure the description check is evaluated first (higher priority).\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_003","query_templated":"Help me with creating a case assignment rule named 'Shoes' by assigning [cases whose description contains the keyword \"shoe\" or whose case number starts with 2] to [shoe case support queue]. [The first condition has higher priority to be evaluated.]","query":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nCreate a rule called Shoes. If either condition matches—description includes “shoe” or the case number starts with 2—send the case to the shoe case support queue. Make sure the description check is evaluated first (higher priority).\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Shoes","entry_conditions":[["Case.Description","contains","shoe"],["Case.CaseNumber","startsWith","2"]],"logic_operator":"OR","assignee":"shoe_case_support_queue","assignee_type":"Queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Shoes","entry_conditions":"cases whose description contains the keyword \"shoe\" or whose case number starts with 2","assignee":"shoe case support queue","sort_order_description":"The first condition has higher priority to be evaluated."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/252"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"needs_review","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_assignment_rule","data_retrieval_spec":[{"type":"metadata","id":"assignment_rule_metadata","params":{"types_and_members":{"AssignmentRules":["Case"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path.","review_flags":["condition_precedence_not_checked"],"review_notes":[{"id":"condition_precedence_not_checked","reason_en":"The query requires the description condition to be evaluated first. This evaluator checks condition membership and Boolean logic, but not condition precedence or rule-entry order.","reason_zh":"指令要求优先判断 description 条件；评分器检查条件是否存在及布尔逻辑，不检查条件优先级或规则条目顺序。","evidence":[{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":109,"line_end":127,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":136,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"}]}],"review_provenance":{"review_file":"scuba-verifier-review.json","review_sha256":"1ed7bd039075f446353e7b562e3016aabcfb34c848ace25fa8a3815b4896e836","revision":"b893e2239b035f93d5753a32557346d15935d596","query_sha256":"401fc7bb04b5da7883d0cd3e0213cec62ed9d43ca329da979607e48a355ba010","ground_truth_sha256":"8d87b96762b0ed3dacb0f901c9ec8489fa00cac7c4fae8310f802058d95676f8","hash_encoding":{"query":"UTF-8 bytes of the exact canonical query string","ground_truth":"UTF-8 JSON, sort_keys=true, ensure_ascii=false, separators=(comma,colon)"},"evidence":[{"source_path":"data/test_zero_shot.json","line_start":10580,"line_end":10601,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10580-L10601","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"}],"review_scope":"static_source_review_not_run"}},"instruction_fingerprint":"9e5383708310f548cf53beee622211280c334faec7eee7caf1c803d98b74752e"} {"id":"scuba:service_002_004","source_id":"scuba","original_id":"service_002_004","kind":"benchmark_task","instruction":"Add an assignment rule named ratings that assigns accounts whose Rating is anything other than Cold to Security User. Set its sort order to 1.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_004","query_templated":"Help me with creating a case assignment rule named 'ratings' by assigning [accounts with ratings not equal to Cold] to [Security User]. Set the sort order to [1].","query":"Add an assignment rule named ratings that assigns accounts whose Rating is anything other than Cold to Security User. Set its sort order to 1.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"ratings","entry_conditions":[["Account.Rating","notEqual","Cold"]],"logic_operator":"","assignee":"security_user","assignee_type":"User","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"ratings","entry_conditions":"accounts with ratings not equal to Cold","assignee":"insightssecurity@","sort_order_description":"Set the sort order to [1]"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/253","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"hard","template_id":"service_002","template_name":"create_case_assignment_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/253"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/253"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_assignment_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_case_assignment_rule","original":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"data":{"objects":{"Group":["groups.json"]}}},"objects":["AssignmentRule"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nAdd an assignment rule named ratings that assigns accounts whose Rating is anything other than Cold to Security User. Set its sort order to 1.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_004","query_templated":"Help me with creating a case assignment rule named 'ratings' by assigning [accounts with ratings not equal to Cold] to [Security User]. Set the sort order to [1].","query":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nAdd an assignment rule named ratings that assigns accounts whose Rating is anything other than Cold to Security User. Set its sort order to 1.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"ratings","entry_conditions":[["Account.Rating","notEqual","Cold"]],"logic_operator":"","assignee":"security_user","assignee_type":"User","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"ratings","entry_conditions":"accounts with ratings not equal to Cold","assignee":"insightssecurity@","sort_order_description":"Set the sort order to [1]"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/253"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"needs_review","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_assignment_rule","data_retrieval_spec":[{"type":"metadata","id":"assignment_rule_metadata","params":{"types_and_members":{"AssignmentRules":["Case"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path.","review_flags":["requested_accounts_checked_case_rules","empty_connector_cannot_pass","numeric_sort_order_not_checked"],"review_notes":[{"id":"requested_accounts_checked_case_rules","reason_en":"The query says to assign accounts, while retrieval fixes AssignmentRules to Case. Account.Rating is checked as a rule criterion; Account ownership is not checked. A related-account condition on a Case rule would not establish that the requested accounts were assigned.","reason_zh":"指令要求分派 accounts，但检索固定为 Case 的 AssignmentRules。Account.Rating 仅作为规则条件核对，不检查 Account 所有者；Case 规则中的关联账户条件不等于完成账户分派。","evidence":[{"source_path":"data/test_zero_shot.json","line_start":10640,"line_end":10678,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10640-L10678","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"},{"source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","line_start":39,"line_end":48,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json#L39-L48","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":83,"line_end":115,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83-L115","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"}]},{"id":"empty_connector_cannot_pass","reason_en":"This record has an empty logic_operator. Only AND and OR reach a successful connector branch; the 0.2-weight connector milestone is therefore false on the normal return path.","reason_zh":"此记录的 logic_operator 为空；只有 AND、OR 能进入连接符成功分支，因此正常返回路径中权重为 0.2 的连接符里程碑必为 false。","evidence":[{"source_path":"data/test_zero_shot.json","line_start":10644,"line_end":10656,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10644-L10656","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":144,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L144-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/master_evaluator.py","line_start":106,"line_end":117,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L106-L117","sha256":"9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"}]},{"id":"numeric_sort_order_not_checked","reason_en":"The query explicitly requires sort order 1. This evaluator never reads the requested sort_order or compares returned rule-entry order.","reason_zh":"指令明确要求 sort order 为 1；评分器未读取要求中的 sort_order，也未比较返回的规则条目顺序。","evidence":[{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":109,"line_end":127,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":136,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"}]}],"review_provenance":{"review_file":"scuba-verifier-review.json","review_sha256":"1ed7bd039075f446353e7b562e3016aabcfb34c848ace25fa8a3815b4896e836","revision":"b893e2239b035f93d5753a32557346d15935d596","query_sha256":"a820723ba77b17be9bd7bf3ad040951feec4cd327533dac6ea82e5b948cd69d9","ground_truth_sha256":"92c9b2e39ace2e26383f27a5b09002900bb79194b6354cb51ff70549011c6138","hash_encoding":{"query":"UTF-8 bytes of the exact canonical query string","ground_truth":"UTF-8 JSON, sort_keys=true, ensure_ascii=false, separators=(comma,colon)"},"evidence":[{"source_path":"data/test_zero_shot.json","line_start":10640,"line_end":10656,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10640-L10656","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"}],"review_scope":"static_source_review_not_run"},"static_score_upper_bound":{"value":0.8,"kind":"normal_return_weight_sum_bound","runtime_score":false,"scope_en":"Static bound for the pinned normal-return path only. ScoreCard requires 1 for completion. This is not a measured run score or a claim that the business request is impossible; exceptions have a separate -1 path.","scope_zh":"仅针对固定版本正常返回路径的静态上限；ScoreCard 需达到 1 才判完成。这不是实测分数，也不代表业务请求不可行；异常使用另一个 -1 路径。","evidence":[{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":97,"line_end":103,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L97-L103","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":130,"line_end":141,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L130-L141","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":144,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L144-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/master_evaluator.py","line_start":13,"line_end":18,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L13-L18","sha256":"9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"},{"source_path":"scuba/phases/evaluation/master_evaluator.py","line_start":106,"line_end":117,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L106-L117","sha256":"9a99e8fa4b4eb32308a857a220bfcff7274b235acc269ac2d7697dfb7a9994e4"}]}},"instruction_fingerprint":"738c492381b74669dcecab22b60087a64e43bac40cd07d7889f4f2db5f53a03d"} {"id":"scuba:service_002_005","source_id":"scuba","original_id":"service_002_005","kind":"benchmark_task","instruction":"Set up a case assignment rule titled Apples. Route cases to the fruit case support queue when both conditions are met: the description contains “apple” and the case number starts with 3. Place this rule at sort order 1.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_005","query_templated":"Help me with creating a case assignment rule named 'Apples' by assigning [cases whose description contains the keyword \"apple\" and whose case number starts with 3] to [fruit case support queue]. Make the sort order [1].","query":"Set up a case assignment rule titled Apples. Route cases to the fruit case support queue when both conditions are met: the description contains “apple” and the case number starts with 3. Place this rule at sort order 1.","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Apples","entry_conditions":[["Case.Description","contains","apple"],["Case.CaseNumber","startsWith","3"]],"logic_operator":"AND","assignee_type":"Queue","assignee":"fruit_case_support_queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Apples","entry_conditions":"cases whose description contains the keyword \"apple\" and whose case number starts with 3","assignee":"fruit case support queue","sort_order_description":"Make the sort order [1]"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/254","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"hard","template_id":"service_002","template_name":"create_case_assignment_rule","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/254"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/254"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_assignment_rule"}],"source_schema_notes":[],"template_registry":{"name":"create_case_assignment_rule","original":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"data":{"objects":{"Group":["groups.json"]}}},"objects":["AssignmentRule"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_assignment_rule","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nSet up a case assignment rule titled Apples. Route cases to the fruit case support queue when both conditions are met: the description contains “apple” and the case number starts with 3. Place this rule at sort order 1.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_002_005","query_templated":"Help me with creating a case assignment rule named 'Apples' by assigning [cases whose description contains the keyword \"apple\" and whose case number starts with 3] to [fruit case support queue]. Make the sort order [1].","query":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry.\n\n## Your ultimate task\n```\nSet up a case assignment rule titled Apples. Route cases to the fruit case support queue when both conditions are met: the description contains “apple” and the case number starts with 3. Place this rule at sort order 1.\n```","human_review_comment":"accept","ground_truth_dict":{"rule_name":"Apples","entry_conditions":[["Case.Description","contains","apple"],["Case.CaseNumber","startsWith","3"]],"logic_operator":"AND","assignee_type":"Queue","assignee":"fruit_case_support_queue","sort_order":1},"difficulty":"hard","query_template_name":"create_case_assignment_rule","query_template_metadata":{"template_id":"service_002","template_string":"Help me with creating a case assignment rule named '{rule_name}' by assigning {entry_conditions} to {assignee}. {sort_order_description}.","prerequisites":{"metadata":{"types_and_members":{"Queue":["fruit_case_support_queue","billing_support_queue","network_support_queue","shoe_case_support_queue"]}}},"objects":[],"metadata_types":["AssignmentRules"]},"instance_dict":{"rule_name":"Apples","entry_conditions":"cases whose description contains the keyword \"apple\" and whose case number starts with 3","assignee":"fruit case support queue","sort_order_description":"Make the sort order [1]"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Help me with creating a case assignment rule named 'Billing Cases Assignment Rule' by assigning cases whose description contains the keyword 'billing' or whose case number starts with 5 to billing support queue. The first condition has higher priority to be evaluated.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Case Assignment Rules” and open it.\n- Create a new rule:\n - Click New, set Rule Name to “Billing Cases Assignment Rule”, and Save.\n- Open the newly created “Billing Cases Assignment Rule”.\n- Add Rule Entry 1 (higher priority):\n - Click New.\n - Sort Order: 1\n - Field: Case: Description\n - Operator: contains\n - Value: billing\n - Assign To: Queue → billing support queue\n - Save.\n- Add Rule Entry 2:\n - Click New.\n - Sort Order: 2\n - Field: Case: Case Number\n - Operator: starts with\n - Value: 5\n - Assign To: Queue → billing support queue\n - Save.\n\nNo hotkeys were used.\nSample 2:\nTASK:Help me with creating a case assignment rule named 'Network Cases Assignment Rule' by assigning cases whose description contains the keyword 'network' and whose case number starts with 7 to network support queue.\nEXPERIENCE:\nSuccessful plan to create the case assignment rule:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “case assignment rules” and open Case Assignment Rules.\n- Click New to create a rule; name it “Network Cases Assignment Rule” and Save.\n- Open the newly created rule, then click New to add a rule entry.\n- Set Sort Order to 1.\n- Add criteria:\n - Case: Description — Operator: contains — Value: network\n - Case: Case Number — Operator: starts with — Value: 7\n- Set assignment to Queue and choose “network support queue.”\n- Save the rule entry."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/254"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"needs_review","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L83","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_assignment_rule","data_retrieval_spec":[{"type":"metadata","id":"assignment_rule_metadata","params":{"types_and_members":{"AssignmentRules":["Case"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path.","review_flags":["numeric_sort_order_not_checked"],"review_notes":[{"id":"numeric_sort_order_not_checked","reason_en":"The query explicitly requires sort order 1. This evaluator never reads the requested sort_order or compares returned rule-entry order.","reason_zh":"指令明确要求 sort order 为 1；评分器未读取要求中的 sort_order，也未比较返回的规则条目顺序。","evidence":[{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":109,"line_end":127,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L109-L127","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"},{"source_path":"scuba/phases/evaluation/milestone_evaluator_service.py","line_start":136,"line_end":169,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L136-L169","sha256":"dbf6f0ff71b7f964f01b7d90c73bdc6a44393a62810e9e719f22f9abff557e1e"}]}],"review_provenance":{"review_file":"scuba-verifier-review.json","review_sha256":"1ed7bd039075f446353e7b562e3016aabcfb34c848ace25fa8a3815b4896e836","revision":"b893e2239b035f93d5753a32557346d15935d596","query_sha256":"5fb2d70d5cc7fe5a41d371039c3bec4813463efac67535c90627c8e89d970296","ground_truth_sha256":"e9e9878da2d781e78991b553c87eceba2e4b8f54c981ef1799afe6d84c3586bf","hash_encoding":{"query":"UTF-8 bytes of the exact canonical query string","ground_truth":"UTF-8 JSON, sort_keys=true, ensure_ascii=false, separators=(comma,colon)"},"evidence":[{"source_path":"data/test_zero_shot.json","line_start":10695,"line_end":10716,"source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json#L10695-L10716","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca"}],"review_scope":"static_source_review_not_run"}},"instruction_fingerprint":"0fa23fd8ebab330f1135bfcc164dac94e120ba3139ab072a638ceffecfac5cc4"} {"id":"scuba:service_003_001","source_id":"scuba","original_id":"service_003_001","kind":"benchmark_task","instruction":"Hey, can you make a case for David Kim about a hardware malfunction? Set the status to 'Escalated', mark the origin as a phone call, set priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', and use the subject 'Hardware Malfunction - Warranty Replacement'.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_001","query_templated":"Please create a case record for David Kim about a hardware malfunction. The case status should be 'Escalated' and the case originated from a phone call. Set the priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', subject to 'Hardware Malfunction - Warranty Replacement'.","query":"Hey, can you make a case for David Kim about a hardware malfunction? Set the status to 'Escalated', mark the origin as a phone call, set priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', and use the subject 'Hardware Malfunction - Warranty Replacement'.","human_review_comment":"accept","ground_truth_dict":{"contact_name":"David Kim","case_status":"Escalated","case_origin":"Phone","priority":"High","case_type":"Mechanical","case_reasoning":"Equipment Design","subject":"Hardware Malfunction - Warranty Replacement"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"Please create a case record for David Kim about a hardware malfunction. The case status should be 'Escalated' and the case originated from a phone call. Set the priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', subject to 'Hardware Malfunction - Warranty Replacement'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/255","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"medium","template_id":"service_003","template_name":"create_case_context_setup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/255"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/255"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_context_setup"}],"source_schema_notes":[],"template_registry":{"name":"create_case_context_setup","original":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["contacts.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nHey, can you make a case for David Kim about a hardware malfunction? Set the status to 'Escalated', mark the origin as a phone call, set priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', and use the subject 'Hardware Malfunction - Warranty Replacement'.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_001","query_templated":"Please create a case record for David Kim about a hardware malfunction. The case status should be 'Escalated' and the case originated from a phone call. Set the priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', subject to 'Hardware Malfunction - Warranty Replacement'.","query":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nHey, can you make a case for David Kim about a hardware malfunction? Set the status to 'Escalated', mark the origin as a phone call, set priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', and use the subject 'Hardware Malfunction - Warranty Replacement'.\n```","human_review_comment":"accept","ground_truth_dict":{"contact_name":"David Kim","case_status":"Escalated","case_origin":"Phone","priority":"High","case_type":"Mechanical","case_reasoning":"Equipment Design","subject":"Hardware Malfunction - Warranty Replacement"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"Please create a case record for David Kim about a hardware malfunction. The case status should be 'Escalated' and the case originated from a phone call. Set the priority to High, type to 'Mechanical', case reasoning to 'Equipment Design', subject to 'Hardware Malfunction - Warranty Replacement'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/255"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L172","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_context_setup","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Priority, Type, Reason, Contact.Name FROM Case WHERE Subject='{subject}' AND Status='{case_status}' AND Origin='{case_origin}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6779a07e93dc211a5713a910111f03afa4063251f846c9a3302a6c81fad34085"} {"id":"scuba:service_003_002","source_id":"scuba","original_id":"service_003_002","kind":"benchmark_task","instruction":"Please log a case for Ronny Chen about a software licensing issue with the following settings: - Status: 'New' - Origin: web form submission - Priority: Medium - Type: 'Electronic' - Case Reasoning: 'Performance' - Subject: 'Software License Key Activation Required'","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_002","query_templated":"We need to create a case record for Ronny Chen regarding his software licensing issue. The case status should be 'New' and the case originated from a web form submission. Set the priority to Medium, type to 'Electronic', case reasoning to 'Performance', subject to 'Software License Key Activation Required'.","query":"Please log a case for Ronny Chen about a software licensing issue with the following settings: - Status: 'New' - Origin: web form submission - Priority: Medium - Type: 'Electronic' - Case Reasoning: 'Performance' - Subject: 'Software License Key Activation Required'","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Ronny Chen","case_status":"New","case_origin":"Web","priority":"Medium","case_type":"Electronic","case_reasoning":"Performance","subject":"Software License Key Activation Required"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Ronny Chen regarding his software licensing issue. The case status should be 'New' and the case originated from a web form submission. Set the priority to Medium, type to 'Electronic', case reasoning to 'Performance', subject to 'Software License Key Activation Required'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/256","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"medium","template_id":"service_003","template_name":"create_case_context_setup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/256"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/256"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_context_setup"}],"source_schema_notes":[],"template_registry":{"name":"create_case_context_setup","original":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["contacts.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease log a case for Ronny Chen about a software licensing issue with the following settings: - Status: 'New' - Origin: web form submission - Priority: Medium - Type: 'Electronic' - Case Reasoning: 'Performance' - Subject: 'Software License Key Activation Required'\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_002","query_templated":"We need to create a case record for Ronny Chen regarding his software licensing issue. The case status should be 'New' and the case originated from a web form submission. Set the priority to Medium, type to 'Electronic', case reasoning to 'Performance', subject to 'Software License Key Activation Required'.","query":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nPlease log a case for Ronny Chen about a software licensing issue with the following settings: - Status: 'New' - Origin: web form submission - Priority: Medium - Type: 'Electronic' - Case Reasoning: 'Performance' - Subject: 'Software License Key Activation Required'\n```","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Ronny Chen","case_status":"New","case_origin":"Web","priority":"Medium","case_type":"Electronic","case_reasoning":"Performance","subject":"Software License Key Activation Required"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Ronny Chen regarding his software licensing issue. The case status should be 'New' and the case originated from a web form submission. Set the priority to Medium, type to 'Electronic', case reasoning to 'Performance', subject to 'Software License Key Activation Required'."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/256"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L172","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_context_setup","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Priority, Type, Reason, Contact.Name FROM Case WHERE Subject='{subject}' AND Status='{case_status}' AND Origin='{case_origin}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"a57456c408c1833f84e3058bdf346eafc41791636cbce792a13ca3cd88a4fcfc"} {"id":"scuba:service_003_003","source_id":"scuba","original_id":"service_003_003","kind":"benchmark_task","instruction":"For Alice Bob, set up a case for her account access issue. Put the case in 'Working' status, note that it originated from an email, set the priority to High, choose type 'Other', set case reasoning to 'Equipment Design', and use the subject 'Account Access Denied - Password Reset Required'.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_003","query_templated":"We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.","query":"For Alice Bob, set up a case for her account access issue. Put the case in 'Working' status, note that it originated from an email, set the priority to High, choose type 'Other', set case reasoning to 'Equipment Design', and use the subject 'Account Access Denied - Password Reset Required'.","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Alice Bob","case_status":"Working","case_origin":"Email","priority":"High","case_type":"Other","case_reasoning":"Equipment Design","subject":"Account Access Denied - Password Reset Required"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/257","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"medium","template_id":"service_003","template_name":"create_case_context_setup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/257"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/257"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_context_setup"}],"source_schema_notes":[],"template_registry":{"name":"create_case_context_setup","original":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["contacts.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor Alice Bob, set up a case for her account access issue. Put the case in 'Working' status, note that it originated from an email, set the priority to High, choose type 'Other', set case reasoning to 'Equipment Design', and use the subject 'Account Access Denied - Password Reset Required'.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_003","query_templated":"We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.","query":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nFor Alice Bob, set up a case for her account access issue. Put the case in 'Working' status, note that it originated from an email, set the priority to High, choose type 'Other', set case reasoning to 'Equipment Design', and use the subject 'Account Access Denied - Password Reset Required'.\n```","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Alice Bob","case_status":"Working","case_origin":"Email","priority":"High","case_type":"Other","case_reasoning":"Equipment Design","subject":"Account Access Denied - Password Reset Required"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/257"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L172","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_context_setup","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Priority, Type, Reason, Contact.Name FROM Case WHERE Subject='{subject}' AND Status='{case_status}' AND Origin='{case_origin}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"398604a4c8fe457302cbf54aeb433f2803631f4facafdf33e1dc581e9fcd1238"} {"id":"scuba:service_003_004","source_id":"scuba","original_id":"service_003_004","kind":"benchmark_task","instruction":"I just got off the phone with Ethan Parker about a network connectivity problem—please file a case. Make the status 'New', priority Medium, type 'Electrical', and case reasoning 'Installation'. The subject should read 'Network Connectivity Issues - Router Configuration'.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_004","query_templated":"Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.","query":"I just got off the phone with Ethan Parker about a network connectivity problem—please file a case. Make the status 'New', priority Medium, type 'Electrical', and case reasoning 'Installation'. The subject should read 'Network Connectivity Issues - Router Configuration'.","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Ethan Parker","case_status":"New","case_origin":"Phone","priority":"Medium","case_type":"Electrical","case_reasoning":"Installation","subject":"Network Connectivity Issues - Router Configuration"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/258","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"medium","template_id":"service_003","template_name":"create_case_context_setup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/258"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/258"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_context_setup"}],"source_schema_notes":[],"template_registry":{"name":"create_case_context_setup","original":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["contacts.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI just got off the phone with Ethan Parker about a network connectivity problem—please file a case. Make the status 'New', priority Medium, type 'Electrical', and case reasoning 'Installation'. The subject should read 'Network Connectivity Issues - Router Configuration'.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_004","query_templated":"Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.","query":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nI just got off the phone with Ethan Parker about a network connectivity problem—please file a case. Make the status 'New', priority Medium, type 'Electrical', and case reasoning 'Installation'. The subject should read 'Network Connectivity Issues - Router Configuration'.\n```","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Ethan Parker","case_status":"New","case_origin":"Phone","priority":"Medium","case_type":"Electrical","case_reasoning":"Installation","subject":"Network Connectivity Issues - Router Configuration"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/258"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L172","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_context_setup","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Priority, Type, Reason, Contact.Name FROM Case WHERE Subject='{subject}' AND Status='{case_status}' AND Origin='{case_origin}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9141ce51cda21f17755a8edf3832f79d78b182bec3121a1db4e94f5e9a030062"} {"id":"scuba:service_003_005","source_id":"scuba","original_id":"service_003_005","kind":"benchmark_task","instruction":"Open a case for Emily Johnson regarding a system performance issue. Keep the status at 'Working', set the origin to a web form submission, priority to Low, type to 'Structural', case reasoning to 'Feedback', and the subject to 'System Performance Degradation - Memory Upgrade Needed'.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_005","query_templated":"We need to create a case record for Emily Johnson regarding her system performance issue. The case status should be 'Working' and the case originated from a web form submission. Set the priority to Low, type to 'Structural', case reasoning to 'Feedback', subject to 'System Performance Degradation - Memory Upgrade Needed'.","query":"Open a case for Emily Johnson regarding a system performance issue. Keep the status at 'Working', set the origin to a web form submission, priority to Low, type to 'Structural', case reasoning to 'Feedback', and the subject to 'System Performance Degradation - Memory Upgrade Needed'.","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Emily Johnson","case_status":"Working","case_origin":"Web","priority":"Low","case_type":"Structural","case_reasoning":"Feedback","subject":"System Performance Degradation - Memory Upgrade Needed"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Emily Johnson regarding her system performance issue. The case status should be 'Working' and the case originated from a web form submission. Set the priority to Low, type to 'Structural', case reasoning to 'Feedback', subject to 'System Performance Degradation - Memory Upgrade Needed'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/259","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"medium","template_id":"service_003","template_name":"create_case_context_setup","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/259"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/259"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_case_context_setup"}],"source_schema_notes":[],"template_registry":{"name":"create_case_context_setup","original":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["contacts.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_case_context_setup","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOpen a case for Emily Johnson regarding a system performance issue. Keep the status at 'Working', set the origin to a web form submission, priority to Low, type to 'Structural', case reasoning to 'Feedback', and the subject to 'System Performance Degradation - Memory Upgrade Needed'.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_003_005","query_templated":"We need to create a case record for Emily Johnson regarding her system performance issue. The case status should be 'Working' and the case originated from a web form submission. Set the priority to Low, type to 'Structural', case reasoning to 'Feedback', subject to 'System Performance Degradation - Memory Upgrade Needed'.","query":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nOpen a case for Emily Johnson regarding a system performance issue. Keep the status at 'Working', set the origin to a web form submission, priority to Low, type to 'Structural', case reasoning to 'Feedback', and the subject to 'System Performance Degradation - Memory Upgrade Needed'.\n```","human_review_comment":"accept","ground_truth_dict":{"contact_name":"Emily Johnson","case_status":"Working","case_origin":"Web","priority":"Low","case_type":"Structural","case_reasoning":"Feedback","subject":"System Performance Degradation - Memory Upgrade Needed"},"difficulty":"medium","query_template_name":"create_case_context_setup","query_template_metadata":{"template_id":"service_003","template_string":"{case_context_setup}","prerequisites":{"data":{"objects":{"Contact":["Contact.json"]}}},"objects":["Case"],"metadata_types":["ValidationRule"]},"instance_dict":{"case_context_setup":"We need to create a case record for Emily Johnson regarding her system performance issue. The case status should be 'Working' and the case originated from a web form submission. Set the priority to Low, type to 'Structural', case reasoning to 'Feedback', subject to 'System Performance Degradation - Memory Upgrade Needed'."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:We need to create a case record for Alice Bob regarding her account access issue. The case status should be 'Working' and the case originated from an email. Set the priority to High, type to 'Other', case reasoning to 'Equipment Design', subject to 'Account Access Denied - Password Reset Required'.\nEXPERIENCE:\nSuccessful plan to create the case:\n\n- Log in to Salesforce.\n- Open App Launcher, search for “cases”, and open the Cases app/list.\n- Click “New Case” to open the case creation form.\n- Set Contact Name: search/select “Alice Bob”.\n- Set required fields via dropdowns:\n - Status: Working\n - Priority: High\n - Case Origin: Email\n - Type: Other\n - Case Reason: Equipment Design\n- Scroll to the Subject field and enter: “Account Access Denied - Password Reset Required”.\n- Click “Save” to create the case.\nSample 2:\nTASK:Please create a case record for Ethan Parker about a network connectivity problem. The case status should be 'New' and the case originated from a phone call. Set the priority to Medium, type to 'Electrical', case reasoning to 'Installation', subject to 'Network Connectivity Issues - Router Configuration'.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “cases,” and open the Cases app/list.\n- Click “New Case.”\n- Set Contact to “Ethan Parker.”\n- Set Type to “Electrical.”\n- Set Case Reason to “Installation.”\n- Set Case Origin to “Phone.”\n- Scroll if needed and enter Subject: “Network Connectivity Issues - Router Configuration.”\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/259"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L172","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_case_context_setup","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Priority, Type, Reason, Contact.Name FROM Case WHERE Subject='{subject}' AND Status='{case_status}' AND Origin='{case_origin}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"6d74e6921c95f2bfffba7c62ac6cf12326257f5a791c3b7617de31911b3b3035"} {"id":"scuba:service_004_001","source_id":"scuba","original_id":"service_004_001","kind":"benchmark_task","instruction":"For the case titled 'Login Issues with Mobile Application'—since it falls under enterprise contract terms that need management sign-off—please switch the status to working and set Potential Liability to Yes.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_001","query_templated":"The case with subject 'Login Issues with Mobile Application' involves enterprise contract terms that require management approval. I need to change its status to working and Potential Liability to Yes.","query":"For the case titled 'Login Issues with Mobile Application'—since it falls under enterprise contract terms that need management sign-off—please switch the status to working and set Potential Liability to Yes.","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Login Issues with Mobile Application","new_status":"Working","other_updates":{"PotentialLiability__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001002","case_context":"involves enterprise contract terms that require management approval","user_profile":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/260","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_004","template_name":"escalate_case_change_owner","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/260"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/260"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/escalate_case_change_owner"}],"source_schema_notes":[],"template_registry":{"name":"escalate_case_change_owner","original":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["cases.json"],"User":["users.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner","differs_from_embedded_fields":["prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nFor the case titled 'Login Issues with Mobile Application'—since it falls under enterprise contract terms that need management sign-off—please switch the status to working and set Potential Liability to Yes.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_001","query_templated":"The case with subject 'Login Issues with Mobile Application' involves enterprise contract terms that require management approval. I need to change its status to working and Potential Liability to Yes.","query":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nFor the case titled 'Login Issues with Mobile Application'—since it falls under enterprise contract terms that need management sign-off—please switch the status to working and set Potential Liability to Yes.\n```","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Login Issues with Mobile Application","new_status":"Working","other_updates":{"PotentialLiability__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001002","case_context":"involves enterprise contract terms that require management approval","user_profile":"Sofia Bennett"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/260"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_escalate_case_change_owner","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Status, Priority, SLAViolation__c, PotentialLiability__c FROM Case WHERE Subject='{case_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9cd06874681977c7fdce3c3a85296a4dc0ac22e62458db83d3b85570be63b45c"} {"id":"scuba:service_004_002","source_id":"scuba","original_id":"service_004_002","kind":"benchmark_task","instruction":"The case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_002","query_templated":"The case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.","query":"The case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.","human_review_comment":"reject; use the original","ground_truth_dict":{"case_subject":"Feature Request: Dark Mode Support","new_status":"Pending","other_updates":{"SLAViolation__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001003","case_context":"requires specialized database expertise that exceeds my current skill level","user_profile":"Michael Johnson"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/261","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_004","template_name":"escalate_case_change_owner","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/261"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/261"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/escalate_case_change_owner"}],"source_schema_notes":[],"template_registry":{"name":"escalate_case_change_owner","original":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["cases.json"],"User":["users.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner","differs_from_embedded_fields":["prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nThe case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_002","query_templated":"The case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.","query":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nThe case with Subject 'Feature Request: Dark Mode Support' requires specialized database expertise that exceeds my current skill level. I need to change its status to pending and SLA Violation to Yes.\n```","human_review_comment":"reject; use the original","ground_truth_dict":{"case_subject":"Feature Request: Dark Mode Support","new_status":"Pending","other_updates":{"SLAViolation__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001003","case_context":"requires specialized database expertise that exceeds my current skill level","user_profile":"Michael Johnson"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/261"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_escalate_case_change_owner","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Status, Priority, SLAViolation__c, PotentialLiability__c FROM Case WHERE Subject='{case_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3415b37b187814a57f57759600cdb8084d12c09291f3becce5aa86fc09450a7e"} {"id":"scuba:service_004_003","source_id":"scuba","original_id":"service_004_003","kind":"benchmark_task","instruction":"The 'Payment Processing Error' case has security compliance implications that call for a security specialist. Please update it to status Working and turn Potential Liability to Yes.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_003","query_templated":"The case with subject 'Payment Processing Error' involves security compliance issues that require specialized security expertise. I need to change its status to Working and Potential Liability to Yes.","query":"The 'Payment Processing Error' case has security compliance implications that call for a security specialist. Please update it to status Working and turn Potential Liability to Yes.","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Payment Processing Error","new_status":"Working","other_updates":{"PotentialLiability__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001005","case_context":"involves security compliance issues that require specialized security expertise","user_profile":"Maria Rodriguez"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/262","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_004","template_name":"escalate_case_change_owner","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/262"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/262"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/escalate_case_change_owner"}],"source_schema_notes":[],"template_registry":{"name":"escalate_case_change_owner","original":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["cases.json"],"User":["users.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner","differs_from_embedded_fields":["prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nThe 'Payment Processing Error' case has security compliance implications that call for a security specialist. Please update it to status Working and turn Potential Liability to Yes.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_003","query_templated":"The case with subject 'Payment Processing Error' involves security compliance issues that require specialized security expertise. I need to change its status to Working and Potential Liability to Yes.","query":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nThe 'Payment Processing Error' case has security compliance implications that call for a security specialist. Please update it to status Working and turn Potential Liability to Yes.\n```","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Payment Processing Error","new_status":"Working","other_updates":{"PotentialLiability__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001005","case_context":"involves security compliance issues that require specialized security expertise","user_profile":"Maria Rodriguez"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/262"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_escalate_case_change_owner","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Status, Priority, SLAViolation__c, PotentialLiability__c FROM Case WHERE Subject='{case_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"cf953eee992be864bc972691db1e192849bb6adb88f3017296af92122657989c"} {"id":"scuba:service_004_004","source_id":"scuba","original_id":"service_004_004","kind":"benchmark_task","instruction":"Escalate the 'Data Synchronization Issue' case, as it’s a complex integration that should be reviewed by a senior developer, and bump the priority to High.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_004","query_templated":"The case with subject 'Data Synchronization Issue' involves complex integration requirements that require senior developer review. I need to change its status to escalated and set the priority to High.","query":"Escalate the 'Data Synchronization Issue' case, as it’s a complex integration that should be reviewed by a senior developer, and bump the priority to High.","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Data Synchronization Issue","new_status":"Escalated","other_updates":{"Priority":"High"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001004","case_context":"involves complex integration requirements that require senior developer review","user_profile":"John Doe"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/263","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_004","template_name":"escalate_case_change_owner","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/263"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/263"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/escalate_case_change_owner"}],"source_schema_notes":[],"template_registry":{"name":"escalate_case_change_owner","original":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["cases.json"],"User":["users.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner","differs_from_embedded_fields":["prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nEscalate the 'Data Synchronization Issue' case, as it’s a complex integration that should be reviewed by a senior developer, and bump the priority to High.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_004","query_templated":"The case with subject 'Data Synchronization Issue' involves complex integration requirements that require senior developer review. I need to change its status to escalated and set the priority to High.","query":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nEscalate the 'Data Synchronization Issue' case, as it’s a complex integration that should be reviewed by a senior developer, and bump the priority to High.\n```","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Data Synchronization Issue","new_status":"Escalated","other_updates":{"Priority":"High"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001004","case_context":"involves complex integration requirements that require senior developer review","user_profile":"John Doe"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/263"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_escalate_case_change_owner","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Status, Priority, SLAViolation__c, PotentialLiability__c FROM Case WHERE Subject='{case_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"68cbd33b04aa4c8416af33d55746c9e8b0e85351e9d611872bff9d42d174b35b"} {"id":"scuba:service_004_005","source_id":"scuba","original_id":"service_004_005","kind":"benchmark_task","instruction":"I’m out of my depth on the case 'Account Billing Discrepancy.' Please move it to escalated, flag SLA Violation as Yes, and raise the Priority to High.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_005","query_templated":"The case with subject 'Account Billing Discrepancy' requires specialized technical expertise that exceeds my current skill level. I need to change its status to escalated and mark SLA Violation as Yes, modifying the Priority to High.","query":"I’m out of my depth on the case 'Account Billing Discrepancy.' Please move it to escalated, flag SLA Violation as Yes, and raise the Priority to High.","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Account Billing Discrepancy","new_status":"Escalated","other_updates":{"Priority":"High","SLAViolation__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001001","case_context":"requires specialized technical expertise that exceeds my current skill level","user_profile":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/264","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Case Management","source_difficulty":"easy","template_id":"service_004","template_name":"escalate_case_change_owner","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/264"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/264"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/escalate_case_change_owner"}],"source_schema_notes":[],"template_registry":{"name":"escalate_case_change_owner","original":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["cases.json"],"User":["users.json"]}}},"objects":["Case"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/escalate_case_change_owner","differs_from_embedded_fields":["prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nI’m out of my depth on the case 'Account Billing Discrepancy.' Please move it to escalated, flag SLA Violation as Yes, and raise the Priority to High.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Case Management","task_id":"service_004_005","query_templated":"The case with subject 'Account Billing Discrepancy' requires specialized technical expertise that exceeds my current skill level. I need to change its status to escalated and mark SLA Violation as Yes, modifying the Priority to High.","query":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save.\n\n## Your ultimate task\n```\nI’m out of my depth on the case 'Account Billing Discrepancy.' Please move it to escalated, flag SLA Violation as Yes, and raise the Priority to High.\n```","human_review_comment":"accept","ground_truth_dict":{"case_subject":"Account Billing Discrepancy","new_status":"Escalated","other_updates":{"Priority":"High","SLAViolation__c":"Yes"}},"difficulty":"easy","query_template_name":"escalate_case_change_owner","query_template_metadata":{"template_id":"service_004","template_string":"The case {case_number} {case_context}. I need to change its status to escalated and change its owner to {user_profile}.","prerequisites":{"data":{"objects":{"Case":["Case.json"],"User":["User.json"]}}},"objects":["Case"],"metadata_types":[]},"instance_dict":{"case_number":"00001001","case_context":"requires specialized technical expertise that exceeds my current skill level","user_profile":"Alice Bob"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:The case with the subject \"Login Issues with Mobile Application\" involves enterprise contract terms that require management approval. I need to change its Status to \"Working\" and Potential Liability to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Cases,” and open the Cases app/object.\n- Locate the case titled “Login Issues with Mobile Application.”\n- Open the row/action dropdown for the case and select Edit.\n- In the edit dialog:\n - Set Status to “Working.”\n - Set Potential Liability to “Yes.”\n- Save the changes.\n\nNotes:\n- No hot-keys were used in this trajectory.\nSample 2:\nTASK:The case with Subject \"Feature Request: Dark Mode Support\" requires specialized database expertise that exceeds my current skill level. I need to change its Status to \"Pending\" and SLA Violation to \"Yes\".\nEXPERIENCE:\nSuccessful plan to update the case:\n\n- Log in to the system.\n- Open App Launcher, search for “Cases”, and open the Cases app.\n- Locate and open the case with Subject “Feature Request: Dark Mode Support”.\n- Open the record’s dropdown menu and select Edit.\n- Set Status to “Pending”.\n- Set SLA Violation to “Yes”.\n- Click Save."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/264"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L221","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_escalate_case_change_owner","data_retrieval_spec":[{"type":"soql","id":"case_info","params":{"query":"SELECT Id, Status, Priority, SLAViolation__c, PotentialLiability__c FROM Case WHERE Subject='{case_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"abb0ee55a45aff9d0637cdb6585d00abcf753d35f9584ef60fcf49e0843993e6"} {"id":"scuba:service_005_001","source_id":"scuba","original_id":"service_005_001","kind":"benchmark_task","instruction":"Per our SLA, Old Balance is eligible for Phone Support that kicks in one week from today and runs for a full year. Please create an entitlement record for that account called Standard Phone Support.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_001","query_templated":"According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.","query":"Per our SLA, Old Balance is eligible for Phone Support that kicks in one week from today and runs for a full year. Please create an entitlement record for that account called Standard Phone Support.","human_review_comment":"accept","ground_truth_dict":{"account_name":"Old Balance","entitlement_name":"Standard Phone Support","support_type":"Phone Support","start_date_relative":"1 week","end_date_relative":"1 year"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Old Balance","support_type":"Phone Support","date_description":"starts a week from today and expires in 1 year","entitlement_name":"Standard Phone Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/265","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_005","template_name":"create_entitlement_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/265"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/265"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_record"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_record","original":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["accounts.json"]}}},"objects":["Entitlement"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nPer our SLA, Old Balance is eligible for Phone Support that kicks in one week from today and runs for a full year. Please create an entitlement record for that account called Standard Phone Support.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_001","query_templated":"According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.","query":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nPer our SLA, Old Balance is eligible for Phone Support that kicks in one week from today and runs for a full year. Please create an entitlement record for that account called Standard Phone Support.\n```","human_review_comment":"accept","ground_truth_dict":{"account_name":"Old Balance","entitlement_name":"Standard Phone Support","support_type":"Phone Support","start_date_relative":"1 week","end_date_relative":"1 year"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Old Balance","support_type":"Phone Support","date_description":"starts a week from today and expires in 1 year","entitlement_name":"Standard Phone Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/265"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L264","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_record","data_retrieval_spec":[{"type":"soql","id":"entitlement_info","params":{"query":"SELECT Id, Type, StartDate, EndDate FROM Entitlement WHERE Name='{entitlement_name}' AND Account.Name='{account_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3782ad54c019d5aa48de45faa5a963ca4ada8c1a4cc9c2fd15569265f3eabe2a"} {"id":"scuba:service_005_002","source_id":"scuba","original_id":"service_005_002","kind":"benchmark_task","instruction":"As outlined in the SLA, Greentrail Logistics should have Web Support starting two weeks from now and continuing for 18 months. Go ahead and create an entitlement record named Standard Web Support.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_002","query_templated":"According to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.","query":"As outlined in the SLA, Greentrail Logistics should have Web Support starting two weeks from now and continuing for 18 months. Go ahead and create an entitlement record named Standard Web Support.","human_review_comment":"accept","ground_truth_dict":{"account_name":"Greentrail Logistics","entitlement_name":"Standard Web Support","support_type":"Web Support","start_date_relative":"2 weeks","end_date_relative":"18 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Greentrail Logistics","support_type":"Web Support","date_description":"starts two weeks from today and expires in 18 months","entitlement_name":"Standard Web Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/266","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_005","template_name":"create_entitlement_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/266"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/266"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_record"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_record","original":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["accounts.json"]}}},"objects":["Entitlement"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nAs outlined in the SLA, Greentrail Logistics should have Web Support starting two weeks from now and continuing for 18 months. Go ahead and create an entitlement record named Standard Web Support.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_002","query_templated":"According to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.","query":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nAs outlined in the SLA, Greentrail Logistics should have Web Support starting two weeks from now and continuing for 18 months. Go ahead and create an entitlement record named Standard Web Support.\n```","human_review_comment":"accept","ground_truth_dict":{"account_name":"Greentrail Logistics","entitlement_name":"Standard Web Support","support_type":"Web Support","start_date_relative":"2 weeks","end_date_relative":"18 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Greentrail Logistics","support_type":"Web Support","date_description":"starts two weeks from today and expires in 18 months","entitlement_name":"Standard Web Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/266"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L264","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_record","data_retrieval_spec":[{"type":"soql","id":"entitlement_info","params":{"query":"SELECT Id, Type, StartDate, EndDate FROM Entitlement WHERE Name='{entitlement_name}' AND Account.Name='{account_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"5dd374238b3c19a750ebe731911670c9dd2cf4814ac833e9d96de22044583e44"} {"id":"scuba:service_005_003","source_id":"scuba","original_id":"service_005_003","kind":"benchmark_task","instruction":"I checked the SLA for Next Horizon Technologies—Phone Support begins in three days and remains active for 1 year plus 30 days. Please add an entitlement record titled Premium Phone Support.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_003","query_templated":"According to the service level agreement (SLA), the Next Horizon Technologies account has the Phone Support entitlement, which starts three days from today and expires in 1 year and 30 days. Create an entitlement record named Premium Phone Support for me.","query":"I checked the SLA for Next Horizon Technologies—Phone Support begins in three days and remains active for 1 year plus 30 days. Please add an entitlement record titled Premium Phone Support.","human_review_comment":"accept","ground_truth_dict":{"account_name":"Next Horizon Technologies","entitlement_name":"Premium Phone Support","support_type":"Phone Support","start_date_relative":"3 days","end_date_relative":"1 year 30 days"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Next Horizon Technologies","support_type":"Phone Support","date_description":"starts three days from today and expires in 1 year and 30 days","entitlement_name":"Premium Phone Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/267","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_005","template_name":"create_entitlement_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/267"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/267"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_record"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_record","original":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["accounts.json"]}}},"objects":["Entitlement"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nI checked the SLA for Next Horizon Technologies—Phone Support begins in three days and remains active for 1 year plus 30 days. Please add an entitlement record titled Premium Phone Support.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_003","query_templated":"According to the service level agreement (SLA), the Next Horizon Technologies account has the Phone Support entitlement, which starts three days from today and expires in 1 year and 30 days. Create an entitlement record named Premium Phone Support for me.","query":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nI checked the SLA for Next Horizon Technologies—Phone Support begins in three days and remains active for 1 year plus 30 days. Please add an entitlement record titled Premium Phone Support.\n```","human_review_comment":"accept","ground_truth_dict":{"account_name":"Next Horizon Technologies","entitlement_name":"Premium Phone Support","support_type":"Phone Support","start_date_relative":"3 days","end_date_relative":"1 year 30 days"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Next Horizon Technologies","support_type":"Phone Support","date_description":"starts three days from today and expires in 1 year and 30 days","entitlement_name":"Premium Phone Support"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/267"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L264","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_record","data_retrieval_spec":[{"type":"soql","id":"entitlement_info","params":{"query":"SELECT Id, Type, StartDate, EndDate FROM Entitlement WHERE Name='{entitlement_name}' AND Account.Name='{account_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"dfffd6acf53859353bc54589e682557316c5fbf0d49c69c7f1944935d4b58a7c"} {"id":"scuba:service_005_004","source_id":"scuba","original_id":"service_005_004","kind":"benchmark_task","instruction":"Per the SLA, White Industries account gets Web Support starting in two weeks from todayt and expiring six months later. Create an entitlement record named Enterprise Chat Support.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_004","query_templated":"According to the service level agreement (SLA), the White Industries account has the Web Support entitlement, which starts in two weeks and expires in 6 months from today. Create an entitlement record named Enterprise Chat Support for me.","query":"Per the SLA, White Industries account gets Web Support starting in two weeks from todayt and expiring six months later. Create an entitlement record named Enterprise Chat Support.","human_review_comment":"edit multiple places","ground_truth_dict":{"account_name":"White Industries","entitlement_name":"Enterprise Chat Support","support_type":"Web Support","start_date_relative":"2 weeks","end_date_relative":"6 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"White Industries","support_type":"Web Support","date_description":"starts in two weeks and expires in 6 months from today","entitlement_name":"Enterprise Chat Support"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/268","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_005","template_name":"create_entitlement_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/268"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/268"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_record"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_record","original":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["accounts.json"]}}},"objects":["Entitlement"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nPer the SLA, White Industries account gets Web Support starting in two weeks from todayt and expiring six months later. Create an entitlement record named Enterprise Chat Support.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_004","query_templated":"According to the service level agreement (SLA), the White Industries account has the Web Support entitlement, which starts in two weeks and expires in 6 months from today. Create an entitlement record named Enterprise Chat Support for me.","query":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nPer the SLA, White Industries account gets Web Support starting in two weeks from todayt and expiring six months later. Create an entitlement record named Enterprise Chat Support.\n```","human_review_comment":"edit multiple places","ground_truth_dict":{"account_name":"White Industries","entitlement_name":"Enterprise Chat Support","support_type":"Web Support","start_date_relative":"2 weeks","end_date_relative":"6 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"White Industries","support_type":"Web Support","date_description":"starts in two weeks and expires in 6 months from today","entitlement_name":"Enterprise Chat Support"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/268"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L264","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_record","data_retrieval_spec":[{"type":"soql","id":"entitlement_info","params":{"query":"SELECT Id, Type, StartDate, EndDate FROM Entitlement WHERE Name='{entitlement_name}' AND Account.Name='{account_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"59b272270f27948e36bc14561762f2f996f2246ed4008eb73d19e4f0b2756d95"} {"id":"scuba:service_005_005","source_id":"scuba","original_id":"service_005_005","kind":"benchmark_task","instruction":"According to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_005","query_templated":"According to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.","query":"According to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.","human_review_comment":"edit multiple places","ground_truth_dict":{"account_name":"Innovate Ltd","entitlement_name":"Premium Email Support Package","support_type":"Phone Support","start_date_relative":"1 week","end_date_relative":"18 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Innovate Ltd","support_type":"phone support type","date_description":"starts in one week from today and expires in 18 months","entitlement_name":"Premium Email Support Package"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/269","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_005","template_name":"create_entitlement_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/269"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/269"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_record"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_record","original":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["accounts.json"]}}},"objects":["Entitlement"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_record","differs_from_embedded_fields":["metadata_types","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nAccording to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_005_005","query_templated":"According to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.","query":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory).\n\n## Your ultimate task\n```\nAccording to the service level agreement (SLA), the Innovate Ltd account has the phone support type entitlement, which starts in one week from today and expires in 18 months. Create an entitlement record named Premium Email Support Package for me.\n```","human_review_comment":"edit multiple places","ground_truth_dict":{"account_name":"Innovate Ltd","entitlement_name":"Premium Email Support Package","support_type":"Phone Support","start_date_relative":"1 week","end_date_relative":"18 months"},"difficulty":"easy","query_template_name":"create_entitlement_record","query_template_metadata":{"template_id":"service_005","template_string":"According to the service level agreement (SLA), the {account_name} account has the {support_type} entitlement, which {date_description}. Create an entitlement record named {entitlement_name} for me.","prerequisites":{"data":{"objects":{"Account":["Account.json"]}}},"objects":["Entitlement"],"metadata_types":["ValidationRule"]},"instance_dict":{"account_name":"Innovate Ltd","support_type":"phone support type","date_description":"starts in one week from today and expires in 18 months","entitlement_name":"Premium Email Support Package"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:According to the service level agreement (SLA), the Old Balance account has the Phone Support entitlement, which starts a week from today and expires in 1 year. Create an entitlement record named Standard Phone Support for me.\nEXPERIENCE:\nSuccessful plan to create the entitlement record\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements app/list.\n- Click New to create a new entitlement.\n- Enter Entitlement Name: “Standard Phone Support.”\n- Set Account Name: search for “Old Balance” and select the matching account from the lookup results.\n- Set Start Date to one week from today (in this run: 08/26/2025).\n- Set End Date to one year after the start date (in this run: 08/26/2026).\n- Click Save.\n\nNotes:\n- No hot-keys were used.\nSample 2:\nTASK:\nAccording to the service level agreement (SLA), the Greentrail Logistics account has the Web Support entitlement, which starts two weeks from today and expires in 18 months. Create an entitlement record named Standard Web Support for me.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce.\n- Open the App Launcher, search for “Entitlements,” and open the Entitlements object.\n- Click New to create a new entitlement.\n- Populate fields:\n - Entitlement Name: Standard Web Support\n - Type: Web Support\n - Account Name: Greentrail Logistics (select the matching account from results)\n - Start Date: Two weeks from today\n - End Date: 18 months after the Start Date\n- Click Save.\n\nNotes:\n- No hotkeys were used.\n- Dates were entered directly in the date fields (MM/DD/YYYY format in the trajectory)."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/269"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L264","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_record","data_retrieval_spec":[{"type":"soql","id":"entitlement_info","params":{"query":"SELECT Id, Type, StartDate, EndDate FROM Entitlement WHERE Name='{entitlement_name}' AND Account.Name='{account_name}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"e8a0434221561bc52df5eacf146bdfcc0310dbe4b5c1227c641ce79a81aa10e1"} {"id":"scuba:service_006_001","source_id":"scuba","original_id":"service_006_001","kind":"benchmark_task","instruction":"Please create a new entitlement process to boost our support standards. It needs four milestones: - \"acknowledgment\" — set this to non-recurring and measure the time until we first acknowledge the case. - \"analysis\" — configure as sequential-recurring and track the time spent analyzing the problem. - \"solution\" — make this non-recurring and capture the time it takes to develop the solution. - \"verification\" — set as independent-recurring and monitor the time required to verify the solution.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_001","query_templated":"To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named 'acknowledgment' should be non-recurring and tracks initial acknowledgment time. The second milestone named 'analysis' is sequential-recurring and tracks problem analysis time. The third milestone named 'solution' is non-recurring and tracks solution development time. The fourth milestone named 'verification' is independent-recurring and tracks solution verification time.","query":"Please create a new entitlement process to boost our support standards. It needs four milestones: - \"acknowledgment\" — set this to non-recurring and measure the time until we first acknowledge the case. - \"analysis\" — configure as sequential-recurring and track the time spent analyzing the problem. - \"solution\" — make this non-recurring and capture the time it takes to develop the solution. - \"verification\" — set as independent-recurring and monitor the time required to verify the solution.","human_review_comment":"accept","ground_truth_dict":{"milestone_count":4,"milestone_name_type_list":[["acknowledgment","none"],["analysis","recursChained"],["solution","none"],["verification","recursIndependently"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"four","milestone_descriptions":"The first milestone named 'acknowledgment' should be non-recurring and tracks initial acknowledgment time. The second milestone named 'analysis' is sequential-recurring and tracks problem analysis time. The third milestone named 'solution' is non-recurring and tracks solution development time. The fourth milestone named 'verification' is independent-recurring and tracks solution verification time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/270","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_006","template_name":"create_milestone","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/270"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/270"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_milestone"}],"source_schema_notes":[],"template_registry":{"name":"create_milestone","original":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nPlease create a new entitlement process to boost our support standards. It needs four milestones: - \"acknowledgment\" — set this to non-recurring and measure the time until we first acknowledge the case. - \"analysis\" — configure as sequential-recurring and track the time spent analyzing the problem. - \"solution\" — make this non-recurring and capture the time it takes to develop the solution. - \"verification\" — set as independent-recurring and monitor the time required to verify the solution.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_001","query_templated":"To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named 'acknowledgment' should be non-recurring and tracks initial acknowledgment time. The second milestone named 'analysis' is sequential-recurring and tracks problem analysis time. The third milestone named 'solution' is non-recurring and tracks solution development time. The fourth milestone named 'verification' is independent-recurring and tracks solution verification time.","query":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nPlease create a new entitlement process to boost our support standards. It needs four milestones: - \"acknowledgment\" — set this to non-recurring and measure the time until we first acknowledge the case. - \"analysis\" — configure as sequential-recurring and track the time spent analyzing the problem. - \"solution\" — make this non-recurring and capture the time it takes to develop the solution. - \"verification\" — set as independent-recurring and monitor the time required to verify the solution.\n```","human_review_comment":"accept","ground_truth_dict":{"milestone_count":4,"milestone_name_type_list":[["acknowledgment","none"],["analysis","recursChained"],["solution","none"],["verification","recursIndependently"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"four","milestone_descriptions":"The first milestone named 'acknowledgment' should be non-recurring and tracks initial acknowledgment time. The second milestone named 'analysis' is sequential-recurring and tracks problem analysis time. The third milestone named 'solution' is non-recurring and tracks solution development time. The fourth milestone named 'verification' is independent-recurring and tracks solution verification time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/270"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L344","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_milestone","data_retrieval_spec":[{"type":"soql","id":"milestone_info","params":{"query":"SELECT Name, RecurrenceType, Description FROM MilestoneType"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"37b5f4bcbb457616766343f93bee3fd5667ba894996e52b7f4926ffd337c25d7"} {"id":"scuba:service_006_002","source_id":"scuba","original_id":"service_006_002","kind":"benchmark_task","instruction":"Can you create a fresh entitlement process with two milestones for service quality? First, add \"assessment\" as independent-recurring to track the initial assessment time. Then add \"delivery\" as non-recurring to measure the final delivery time.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_002","query_templated":"To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named 'assessment' should be independent-recurring and tracks initial assessment time. The second milestone named 'delivery' is non-recurring and tracks final delivery time.","query":"Can you create a fresh entitlement process with two milestones for service quality? First, add \"assessment\" as independent-recurring to track the initial assessment time. Then add \"delivery\" as non-recurring to measure the final delivery time.","human_review_comment":"accept","ground_truth_dict":{"milestone_count":2,"milestone_name_type_list":[["assessment","recursIndependently"],["delivery","none"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"two","milestone_descriptions":"The first milestone named 'assessment' should be independent-recurring and tracks initial assessment time. The second milestone named 'delivery' is non-recurring and tracks final delivery time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/271","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_006","template_name":"create_milestone","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/271"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/271"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_milestone"}],"source_schema_notes":[],"template_registry":{"name":"create_milestone","original":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nCan you create a fresh entitlement process with two milestones for service quality? First, add \"assessment\" as independent-recurring to track the initial assessment time. Then add \"delivery\" as non-recurring to measure the final delivery time.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_002","query_templated":"To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named 'assessment' should be independent-recurring and tracks initial assessment time. The second milestone named 'delivery' is non-recurring and tracks final delivery time.","query":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nCan you create a fresh entitlement process with two milestones for service quality? First, add \"assessment\" as independent-recurring to track the initial assessment time. Then add \"delivery\" as non-recurring to measure the final delivery time.\n```","human_review_comment":"accept","ground_truth_dict":{"milestone_count":2,"milestone_name_type_list":[["assessment","recursIndependently"],["delivery","none"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"two","milestone_descriptions":"The first milestone named 'assessment' should be independent-recurring and tracks initial assessment time. The second milestone named 'delivery' is non-recurring and tracks final delivery time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/271"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L344","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_milestone","data_retrieval_spec":[{"type":"soql","id":"milestone_info","params":{"query":"SELECT Name, RecurrenceType, Description FROM MilestoneType"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"96ae324ef51ec491b3d876a7d9bb8ede75ecc3de6b5aae3a2d765c7ac0aef990"} {"id":"scuba:service_006_003","source_id":"scuba","original_id":"service_006_003","kind":"benchmark_task","instruction":"To improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_003","query_templated":"To improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.","query":"To improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.","human_review_comment":"reject; use original","ground_truth_dict":{"milestone_count":3,"milestone_name_type_list":[["first response","none"],["escalation review","recursChained"],["resolution time","recursIndependently"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"three","milestone_descriptions":"The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/272","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_006","template_name":"create_milestone","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/272"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/272"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_milestone"}],"source_schema_notes":[],"template_registry":{"name":"create_milestone","original":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nTo improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_003","query_templated":"To improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.","query":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nTo improve the customer service quality, we defined a new entitlement process with three milestones. The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"milestone_count":3,"milestone_name_type_list":[["first response","none"],["escalation review","recursChained"],["resolution time","recursIndependently"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"three","milestone_descriptions":"The first milestone named 'first response' should be non-recurring and tracks initial response time. The second milestone named 'escalation review' is sequential-recurring and tracks supervisor review time. The third milestone named 'resolution time' is independent-recurring and tracks final resolution time"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/272"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L344","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_milestone","data_retrieval_spec":[{"type":"soql","id":"milestone_info","params":{"query":"SELECT Name, RecurrenceType, Description FROM MilestoneType"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"0933097663b81c43b8f688b16aca519c14c34eec7e393122ffaae71715af8fa2"} {"id":"scuba:service_006_004","source_id":"scuba","original_id":"service_006_004","kind":"benchmark_task","instruction":"Set up an entitlement process that has just one milestone: call it \"completion.\" It must be sequential-recurring and should track the final completion time.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_004","query_templated":"To improve the customer service quality, we defined a new entitlement process with one milestone. The milestone named 'completion' should be sequential-recurring and tracks final completion time.","query":"Set up an entitlement process that has just one milestone: call it \"completion.\" It must be sequential-recurring and should track the final completion time.","human_review_comment":"accept","ground_truth_dict":{"milestone_count":1,"milestone_name_type_list":[["completion","recursChained"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"one","milestone_descriptions":"The milestone named 'completion' should be sequential-recurring and tracks final completion time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/273","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_006","template_name":"create_milestone","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/273"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/273"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_milestone"}],"source_schema_notes":[],"template_registry":{"name":"create_milestone","original":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nSet up an entitlement process that has just one milestone: call it \"completion.\" It must be sequential-recurring and should track the final completion time.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_004","query_templated":"To improve the customer service quality, we defined a new entitlement process with one milestone. The milestone named 'completion' should be sequential-recurring and tracks final completion time.","query":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nSet up an entitlement process that has just one milestone: call it \"completion.\" It must be sequential-recurring and should track the final completion time.\n```","human_review_comment":"accept","ground_truth_dict":{"milestone_count":1,"milestone_name_type_list":[["completion","recursChained"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"one","milestone_descriptions":"The milestone named 'completion' should be sequential-recurring and tracks final completion time"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/273"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L344","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_milestone","data_retrieval_spec":[{"type":"soql","id":"milestone_info","params":{"query":"SELECT Name, RecurrenceType, Description FROM MilestoneType"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4788f0a16a7ad749d0f9bb61eecb67c7e6aecb65dd327da68a7a4666cbfa29b8"} {"id":"scuba:service_006_005","source_id":"scuba","original_id":"service_006_005","kind":"benchmark_task","instruction":"Please build a new entitlement process with two milestones. Start with \"initial response\" configured as sequential-recurring to time how long we have to provide the first reply to a customer inquiry. Then add \"case closure\" as a non-recurring milestone to track the maximum time allowed to fully resolve and close the case.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_005","query_templated":"To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named 'initial response' should be sequential-recurring and it tracks the time given to provide the first response to a customer inquiry. The second non-recurring milestone named 'case closure' aims to track the maximum time allowed to fully resolve and close a customer case.","query":"Please build a new entitlement process with two milestones. Start with \"initial response\" configured as sequential-recurring to time how long we have to provide the first reply to a customer inquiry. Then add \"case closure\" as a non-recurring milestone to track the maximum time allowed to fully resolve and close the case.","human_review_comment":"accept","ground_truth_dict":{"milestone_count":2,"milestone_name_type_list":[["initial response","recursChained"],["case closure","none"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"two","milestone_descriptions":"The first milestone named 'initial response' should be sequential-recurring and it tracks the time given to provide the first response to a customer inquiry. The second non-recurring milestone named 'case closure' aims to track the maximum time allowed to fully resolve and close a customer case"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/274","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"easy","template_id":"service_006","template_name":"create_milestone","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/274"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/274"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_milestone"}],"source_schema_notes":[],"template_registry":{"name":"create_milestone","original":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_milestone","differs_from_embedded_fields":["metadata_types"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nPlease build a new entitlement process with two milestones. Start with \"initial response\" configured as sequential-recurring to time how long we have to provide the first reply to a customer inquiry. Then add \"case closure\" as a non-recurring milestone to track the maximum time allowed to fully resolve and close the case.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_006_005","query_templated":"To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named 'initial response' should be sequential-recurring and it tracks the time given to provide the first response to a customer inquiry. The second non-recurring milestone named 'case closure' aims to track the maximum time allowed to fully resolve and close a customer case.","query":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first.\n\n## Your ultimate task\n```\nPlease build a new entitlement process with two milestones. Start with \"initial response\" configured as sequential-recurring to time how long we have to provide the first reply to a customer inquiry. Then add \"case closure\" as a non-recurring milestone to track the maximum time allowed to fully resolve and close the case.\n```","human_review_comment":"accept","ground_truth_dict":{"milestone_count":2,"milestone_name_type_list":[["initial response","recursChained"],["case closure","none"]]},"difficulty":"easy","query_template_name":"create_milestone","query_template_metadata":{"template_id":"service_006","template_string":"To improve the customer service quality, we defined a new entitlement process with {milestone_count} milestones. {milestone_descriptions}.","prerequisites":{},"objects":["MilestoneType"],"metadata_types":["EntitlementProcess"]},"instance_dict":{"milestone_count":"two","milestone_descriptions":"The first milestone named 'initial response' should be sequential-recurring and it tracks the time given to provide the first response to a customer inquiry. The second non-recurring milestone named 'case closure' aims to track the maximum time allowed to fully resolve and close a customer case"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:To improve the customer service quality, we defined a new entitlement process with four milestones. The first milestone named \"Acknowledgement\" should be \"non-recurring\" and \"Tracks initial acknowledgement time\". The second milestone named \"Analysis\" is \"sequential-recurring\" and \"Tracks problem analysis time\". The third milestone named \"Solution\" is \"non-recurring\" and \"Tracks solution development time\". The fourth milestone named \"Verification\" is \"independent-recurring\" and \"Tracks solution verification time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce, open Setup.\n- In Quick Find, type “Milestones” and open the Milestones page.\n- Create milestones as follows:\n - Acknowledgement:\n - Name: Acknowledgement\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks initial acknowledgement time\n - Save & New\n - Analysis:\n - Name: Analysis\n - Recurrence Type: Sequential (sequential-recurring)\n - Description: Tracks problem analysis time\n - Save & New\n - Solution:\n - Name: Solution\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks solution development time\n - Save & New\n - Verification:\n - Name: Verification\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks solution verification time\n - Save\n\nNotes:\n- Save & New was used to streamline creation for the first three milestones; Save used for the last.\n- No hotkeys were used.\nSample 2:\nTASK:To improve the customer service quality, we defined a new entitlement process with two milestones. The first milestone named \"Assessment\" should be \"independent-recurring\" and \"Tracks initial assessment time\". The second milestone named \"Delivery\" is \"non-recurring\" and \"Tracks final delivery time\".\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to Salesforce and open Setup.\n- In Quick Find, search for “Entitlement Management” and go to “Milestones.”\n- Create milestone “Assessment”:\n - Name: Assessment\n - Recurrence Type: Independent (independent-recurring)\n - Description: Tracks initial assessment time\n - Click “Save & New”\n- Create milestone “Delivery”:\n - Name: Delivery\n - Recurrence Type: No Recurrence (non-recurring)\n - Description: Tracks final delivery time\n - Click “Save”\n\nNotes:\n- No hot-keys were used.\n- “Save & New” was used to efficiently add the second milestone after the first."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/274"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L344","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_milestone","data_retrieval_spec":[{"type":"soql","id":"milestone_info","params":{"query":"SELECT Name, RecurrenceType, Description FROM MilestoneType"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"92eae89208e048349130e7783d847dffb91e72ac664dc7d79c6e095406a28277"} {"id":"scuba:service_007_001","source_id":"scuba","original_id":"service_007_001","kind":"benchmark_task","instruction":"Please set up an entitlement process called “Priority Customer Support” for Case records. Start the process based on the Case Created Date and have it end when the case is closed. Add a milestone named “first response to priority customers” that gives agents 30 minutes to send the first response to priority customers.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_001","query_templated":"Create an entitlement process with the name Priority Customer Support for the Case records. The case should enter this process based on case created date and exit when it is closed. Include the milestone 'first response to priority customers', which gives the agent 30 mins to provide first response to priority customers.","query":"Please set up an entitlement process called “Priority Customer Support” for Case records. Start the process based on the Case Created Date and have it end when the case is closed. Add a milestone named “first response to priority customers” that gives agents 30 minutes to send the first response to priority customers.","human_review_comment":"accept","ground_truth_dict":{"process_name":"Priority Customer Support","record_type":"Case","milestones":[{"milestoneName":"first response to priority customers"}],"milestone_time_trigger":30},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"Priority Customer Support","record_type":"Case","entry_condition":"case should enter this process based on case created date and exist when it is closed","milestone_details":"the milestone 'first response to priority customers', which gives the agent 30 mins to provide first response to priority customers","violation_action":"Add the warning action to set time trigger 5 minutes before first response to priority customers"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/275","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"hard","template_id":"service_007","template_name":"create_entitlement_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/275"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/275"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_process"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_process","original":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["milestones.json"]}}},"objects":["SlaProcess"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease set up an entitlement process called “Priority Customer Support” for Case records. Start the process based on the Case Created Date and have it end when the case is closed. Add a milestone named “first response to priority customers” that gives agents 30 minutes to send the first response to priority customers.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_001","query_templated":"Create an entitlement process with the name Priority Customer Support for the Case records. The case should enter this process based on case created date and exit when it is closed. Include the milestone 'first response to priority customers', which gives the agent 30 mins to provide first response to priority customers.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nPlease set up an entitlement process called “Priority Customer Support” for Case records. Start the process based on the Case Created Date and have it end when the case is closed. Add a milestone named “first response to priority customers” that gives agents 30 minutes to send the first response to priority customers.\n```","human_review_comment":"accept","ground_truth_dict":{"process_name":"Priority Customer Support","record_type":"Case","milestones":[{"milestoneName":"first response to priority customers"}],"milestone_time_trigger":30},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"Priority Customer Support","record_type":"Case","entry_condition":"case should enter this process based on case created date and exist when it is closed","milestone_details":"the milestone 'first response to priority customers', which gives the agent 30 mins to provide first response to priority customers","violation_action":"Add the warning action to set time trigger 5 minutes before first response to priority customers"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/275"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L365","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_process","data_retrieval_spec":[{"type":"metadata","id":"entitlement_criteria_check","params":{"types_and_members":{"EntitlementProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"728af0b3e8374d5e1a288c9a86ad7e3a471bff1513c397f94151cf7f6ed5c231"} {"id":"scuba:service_007_002","source_id":"scuba","original_id":"service_007_002","kind":"benchmark_task","instruction":"Create an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_002","query_templated":"Create an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.","query":"Create an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.","human_review_comment":"reject; use original","ground_truth_dict":{"process_name":"Premium Support Process","record_type":"Case","milestones":[{"milestoneName":"first response to priority customers"}],"milestone_time_trigger":15},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"Premium Support Process","record_type":"Case","entry_condition":"case should enter this process based on case created date and exist when it is closed","milestone_details":"the milestone 'first response to priority customers', which gives the agent 60 mins to provide initial response to premium customers","violation_action":"Add the warning action for the 'first response to priority customers' milestone to send email alert 15 minutes before deadline. Use 'first_response_to_premium_customers' as the email name, and email template 'Support: Case Response', with recipient user Alice Bob."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/276","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"hard","template_id":"service_007","template_name":"create_entitlement_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/276"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/276"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_process"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_process","original":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["milestones.json"]}}},"objects":["SlaProcess"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_002","query_templated":"Create an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate an entitlement process with the name Premium Support Process for the Case records. The case should enter this process based on case created date and exist when it is closed. Include the milestone 'first response to priority customers', which gives the agent 15 mins to provide initial response to premium customers.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"process_name":"Premium Support Process","record_type":"Case","milestones":[{"milestoneName":"first response to priority customers"}],"milestone_time_trigger":15},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"Premium Support Process","record_type":"Case","entry_condition":"case should enter this process based on case created date and exist when it is closed","milestone_details":"the milestone 'first response to priority customers', which gives the agent 60 mins to provide initial response to premium customers","violation_action":"Add the warning action for the 'first response to priority customers' milestone to send email alert 15 minutes before deadline. Use 'first_response_to_premium_customers' as the email name, and email template 'Support: Case Response', with recipient user Alice Bob."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/276"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L365","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_process","data_retrieval_spec":[{"type":"metadata","id":"entitlement_criteria_check","params":{"types_and_members":{"EntitlementProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2efacec0dbb4ddab2af27c46a28d0ba3cbb8045404d4aa371ef29b775e31daa2"} {"id":"scuba:service_007_003","source_id":"scuba","original_id":"service_007_003","kind":"benchmark_task","instruction":"I need an entitlement process called “case email” for Case records. Kick it off using the Case Created Date and wrap it up when the case closes. Add the “Escalate case” milestone that checks whether the Case Contact Email contains “salesforce,” and set its time trigger to 1 minute.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_003","query_templated":"Create an entitlement process with the name case email for the Case records. The case should enter this process based on case created date and exit when it is closed. Include the milestone 'Escalate case' which checks if the Case Contact Email contains salesforce. Set the time trigger to 1 minute.","query":"I need an entitlement process called “case email” for Case records. Kick it off using the Case Created Date and wrap it up when the case closes. Add the “Escalate case” milestone that checks whether the Case Contact Email contains “salesforce,” and set its time trigger to 1 minute.","human_review_comment":"accept","ground_truth_dict":{"process_name":"case email","record_type":"Case","milestones":[{"milestoneName":"Escalate case","criteria":[["Contact.Email","contains","salesforce"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"case email","record_type":"Case","entry_condition":"type [Case] called [case email] with [one] milestone","milestone_details":"The milestone name should be [Escalate case] and checks if the [Case: Contact Email contains salesforce]","violation_action":"Set the time trigger to [1] minute"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/277","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"hard","template_id":"service_007","template_name":"create_entitlement_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/277"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/277"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_process"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_process","original":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["milestones.json"]}}},"objects":["SlaProcess"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI need an entitlement process called “case email” for Case records. Kick it off using the Case Created Date and wrap it up when the case closes. Add the “Escalate case” milestone that checks whether the Case Contact Email contains “salesforce,” and set its time trigger to 1 minute.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_003","query_templated":"Create an entitlement process with the name case email for the Case records. The case should enter this process based on case created date and exit when it is closed. Include the milestone 'Escalate case' which checks if the Case Contact Email contains salesforce. Set the time trigger to 1 minute.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nI need an entitlement process called “case email” for Case records. Kick it off using the Case Created Date and wrap it up when the case closes. Add the “Escalate case” milestone that checks whether the Case Contact Email contains “salesforce,” and set its time trigger to 1 minute.\n```","human_review_comment":"accept","ground_truth_dict":{"process_name":"case email","record_type":"Case","milestones":[{"milestoneName":"Escalate case","criteria":[["Contact.Email","contains","salesforce"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"case email","record_type":"Case","entry_condition":"type [Case] called [case email] with [one] milestone","milestone_details":"The milestone name should be [Escalate case] and checks if the [Case: Contact Email contains salesforce]","violation_action":"Set the time trigger to [1] minute"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/277"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L365","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_process","data_retrieval_spec":[{"type":"metadata","id":"entitlement_criteria_check","params":{"types_and_members":{"EntitlementProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"557f58d75e34b7d6b76e17a7eb37b7a0f045c52e5130f2d006a5688c8e35b9bc"} {"id":"scuba:service_007_004","source_id":"scuba","original_id":"service_007_004","kind":"benchmark_task","instruction":"Create an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_004","query_templated":"Create an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.","query":"Create an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.","human_review_comment":"reject; use original","ground_truth_dict":{"process_name":"priority incident","record_type":"Incident","milestones":[{"milestoneName":"close case","criteria":[["Incident.IsClosed","equals","true"]]},{"milestoneName":"first response to priority customers","criteria":[["User.AboutMe","notContain","manager"],["Incident.Priority","notEqual","High"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"priority incident","record_type":"Incident","entry_condition":"type [Incident] called [priority incident] with [two] milestones","milestone_details":"The first milestone name should be [close case] and checks if the [Incident: Incident closed equals True]. Set the order to 1. The second milestone name should be [first response to customer] and checks if [current user: about me does not contain manager] and [incident: priority not equal to High]","violation_action":"Set the time trigger to [1] minute for each"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/278","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"hard","template_id":"service_007","template_name":"create_entitlement_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/278"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/278"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_process"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_process","original":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["milestones.json"]}}},"objects":["SlaProcess"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_004","query_templated":"Create an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nCreate an entitlement process with the name priority incident for the Incident records. The incident should enter this process based on incident created date and exit when it is closed. Include the milestones 'close case' that checks if the [Incident: Incident closed equals True] and 'first response to priority customers' which checks if [current user: about me does not contain manager] and [incident: priority not equal to High]. Set the time trigger to 1 minute for each milestone.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"process_name":"priority incident","record_type":"Incident","milestones":[{"milestoneName":"close case","criteria":[["Incident.IsClosed","equals","true"]]},{"milestoneName":"first response to priority customers","criteria":[["User.AboutMe","notContain","manager"],["Incident.Priority","notEqual","High"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"priority incident","record_type":"Incident","entry_condition":"type [Incident] called [priority incident] with [two] milestones","milestone_details":"The first milestone name should be [close case] and checks if the [Incident: Incident closed equals True]. Set the order to 1. The second milestone name should be [first response to customer] and checks if [current user: about me does not contain manager] and [incident: priority not equal to High]","violation_action":"Set the time trigger to [1] minute for each"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/278"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L365","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_process","data_retrieval_spec":[{"type":"metadata","id":"entitlement_criteria_check","params":{"types_and_members":{"EntitlementProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"f08f39060f0ad748b55f8b0ec6b8d18558ff3aaa4320465aea067a794354d457"} {"id":"scuba:service_007_005","source_id":"scuba","original_id":"service_007_005","kind":"benchmark_task","instruction":"Configure an entitlement process called “order amount” for Work Order records. Begin tracking from the Work Order Created Date and end the process at closure. Add a milestone named “Escalate case” with conditions [work order: total price greater than 5000], [work order: tax less than 400], and [current user: country starts with U], and set its time trigger to 1 minute.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_005","query_templated":"Create an entitlement process with the name order amount for the Work Order records. The work order should enter this process based on work order created date and exit when it is closed. Include the milestone 'Escalate case' which checks if the [work order: total price greater than 5000] and [work order: tax less than 400] and [current user: country starts with U]. Set the time trigger to 1 minute.","query":"Configure an entitlement process called “order amount” for Work Order records. Begin tracking from the Work Order Created Date and end the process at closure. Add a milestone named “Escalate case” with conditions [work order: total price greater than 5000], [work order: tax less than 400], and [current user: country starts with U], and set its time trigger to 1 minute.","human_review_comment":"accept","ground_truth_dict":{"process_name":"order amount","record_type":"WorkOrder","milestones":[{"milestoneName":"Escalate Case","criteria":[["WorkOrder.TotalPrice","greaterThan","5000"],["WorkOrder.Tax","lessThan","400"],["User.Country","startsWith","U"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"order amount","record_type":"Work Order","entry_condition":"type [Work Order] called [order amount] with [one] milestones","milestone_details":"The milestone name should be [Escalate case] and checks if the [work order: total price greater than 5000] and [work order: tax less than 400] and [current user: country starts with U]","violation_action":"Set the time trigger to [1] minute"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/279","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Entitlements","source_difficulty":"hard","template_id":"service_007","template_name":"create_entitlement_process","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/279"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/279"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_entitlement_process"}],"source_schema_notes":[],"template_registry":{"name":"create_entitlement_process","original":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["milestones.json"]}}},"objects":["SlaProcess"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_entitlement_process","differs_from_embedded_fields":["metadata_types","objects","prerequisites"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nConfigure an entitlement process called “order amount” for Work Order records. Begin tracking from the Work Order Created Date and end the process at closure. Add a milestone named “Escalate case” with conditions [work order: total price greater than 5000], [work order: tax less than 400], and [current user: country starts with U], and set its time trigger to 1 minute.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Entitlements","task_id":"service_007_005","query_templated":"Create an entitlement process with the name order amount for the Work Order records. The work order should enter this process based on work order created date and exit when it is closed. Include the milestone 'Escalate case' which checks if the [work order: total price greater than 5000] and [work order: tax less than 400] and [current user: country starts with U]. Set the time trigger to 1 minute.","query":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nConfigure an entitlement process called “order amount” for Work Order records. Begin tracking from the Work Order Created Date and end the process at closure. Add a milestone named “Escalate case” with conditions [work order: total price greater than 5000], [work order: tax less than 400], and [current user: country starts with U], and set its time trigger to 1 minute.\n```","human_review_comment":"accept","ground_truth_dict":{"process_name":"order amount","record_type":"WorkOrder","milestones":[{"milestoneName":"Escalate Case","criteria":[["WorkOrder.TotalPrice","greaterThan","5000"],["WorkOrder.Tax","lessThan","400"],["User.Country","startsWith","U"]]}],"milestone_time_trigger":1},"difficulty":"hard","query_template_name":"create_entitlement_process","query_template_metadata":{"template_id":"service_007","template_string":"Create an entitlement process with the name {process_name} for the {record_type} records. The {entry_condition}. Include {milestone_details}. {violation_action}.","prerequisites":{"data":{"objects":{"MilestoneType":["MilestoneType.json"]}}},"objects":[],"metadata_types":["EntitlementProcess"]},"instance_dict":{"process_name":"order amount","record_type":"Work Order","entry_condition":"type [Work Order] called [order amount] with [one] milestones","milestone_details":"The milestone name should be [Escalate case] and checks if the [work order: total price greater than 5000] and [work order: tax less than 400] and [current user: country starts with U]","violation_action":"Set the time trigger to [1] minute"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create an entitlement process with the name \"Priority Customer Support\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 30 minutes to provide first response to priority customers.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in, open Setup.\n- In Quick Find, search for and open “Entitlement Processes”.\n- Click “New Entitlement Process”.\n- Select the Case object for the entitlement process and click “Next”.\n- Enter Entitlement Process Name: “Priority Customer Support”.\n- Leave default timing: case enters based on Created Date and exits when Case is closed.\n- Save the entitlement process.\n- Click “New Milestone”.\n- Set Milestone Name: “first response to priority customers”.\n- Set Order: 1.\n- Set Time Trigger: 30 minutes.\n- Save the milestone.\nSample 2:\nTASK:Create an entitlement process with the name \"Premium Support Process\" for the Case records. The case should enter this process based on the case created date and exit when it is closed. Include the milestone \"first response to priority customers\" at order 1, which gives the agent 15 minutes to provide initial response to premium customers.\nEXPERIENCE:\nSuccessful plan summary:\n\n- Log in to Salesforce and open Setup.\n- Use Quick Find to search for “Entitlement Processes” and open it.\n- Click “New Entitlement Process,” select Case as the process object in the wizard, and click Next.\n- Enter Entitlement Process Name: “Premium Support Process.”\n- Configure the process to start based on Case Created Date and exit when the Case is closed, then Save.\n- Click “New Milestone” to add a milestone to the process:\n - Milestone Name: “first response to priority customers”\n - Order: 1\n - Time Trigger: 15 minutes\n - Save\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/279"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L365","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_entitlement_process","data_retrieval_spec":[{"type":"metadata","id":"entitlement_criteria_check","params":{"types_and_members":{"EntitlementProcess":["*"]}}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"943eb5730f0a1778e23532e0cee1583f679d2bb368982d199cfc8e3be6f788ec"} {"id":"scuba:service_010_001","source_id":"scuba","original_id":"service_010_001","kind":"benchmark_task","instruction":"Create a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_001","query_templated":"Create a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".","query":"Create a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Headlight Replacement","article_published":"Online","question_desc":"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?","answer_desc":"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Profile":["Admin"],"Layout":["Knowledge__kav-Knowledge Layout"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Headlight Replacement","question_desc":"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?","answer_desc":"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/280","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Knowledge Article","source_difficulty":"easy","template_id":"service_010","template_name":"create_knowledge_article","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/280"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/280"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_knowledge_article"}],"source_schema_notes":[],"template_registry":{"name":"create_knowledge_article","original":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be '{article_title}' and the article detail should be '{article_content}'.","prerequisites":{},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article","differs_from_embedded_fields":["objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_001","query_templated":"Create a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be Headlight Replacement with the question description be \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\".\n```","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Headlight Replacement","article_published":"Online","question_desc":"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?","answer_desc":"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"CustomTab":["VoiceCallTranscript__c","Knowledge__kav"],"Profile":["Admin"],"Layout":["Knowledge__kav-Knowledge Layout"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Headlight Replacement","question_desc":"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?","answer_desc":"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/280"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L506","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_knowledge_article","data_retrieval_spec":[{"type":"soql","id":"knowledge_article_info","params":{"query":"SELECT Id, PublishStatus, Question__c, FAQ_Answer__c FROM Knowledge__kav WHERE Title='{article_title}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"9bc4832dd72dc03d3ec528269197f811b7a7cdf2de721a19ecd58b78a54fd8a7"} {"id":"scuba:service_010_002","source_id":"scuba","original_id":"service_010_002","kind":"benchmark_task","instruction":"Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_002","query_templated":"Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"","query":"Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"The Science Behind Sleep and Productivity","article_published":"Online","question_desc":"How does sleep quality impact workplace productivity and cognitive function?","answer_desc":"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"The Science Behind Sleep and Productivity","question_desc":"How does sleep quality impact workplace productivity and cognitive function?","answer_desc":"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/281","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Knowledge Article","source_difficulty":"easy","template_id":"service_010","template_name":"create_knowledge_article","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/281"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/281"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_knowledge_article"}],"source_schema_notes":[],"template_registry":{"name":"create_knowledge_article","original":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be '{article_title}' and the article detail should be '{article_content}'.","prerequisites":{},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article","differs_from_embedded_fields":["objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_002","query_templated":"Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description be \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\n```","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"The Science Behind Sleep and Productivity","article_published":"Online","question_desc":"How does sleep quality impact workplace productivity and cognitive function?","answer_desc":"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"The Science Behind Sleep and Productivity","question_desc":"How does sleep quality impact workplace productivity and cognitive function?","answer_desc":"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/281"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L506","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_knowledge_article","data_retrieval_spec":[{"type":"soql","id":"knowledge_article_info","params":{"query":"SELECT Id, PublishStatus, Question__c, FAQ_Answer__c FROM Knowledge__kav WHERE Title='{article_title}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"29e8b1ac2f644340df2a82d883644eb75171e68e4159c22bc70e724121bd52ab"} {"id":"scuba:service_010_003","source_id":"scuba","original_id":"service_010_003","kind":"benchmark_task","instruction":"Create a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_003","query_templated":"Create a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"","query":"Create a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Best Practices for Remote Team Collaboration","article_published":"Online","question_desc":"What strategies help remote teams stay productive and connected?","answer_desc":"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Best Practices for Remote Team Collaboration","question_desc":"What strategies help remote teams stay productive and connected?","answer_desc":"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/282","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Knowledge Article","source_difficulty":"easy","template_id":"service_010","template_name":"create_knowledge_article","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/282"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/282"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_knowledge_article"}],"source_schema_notes":[],"template_registry":{"name":"create_knowledge_article","original":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be '{article_title}' and the article detail should be '{article_content}'.","prerequisites":{},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article","differs_from_embedded_fields":["objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_003","query_templated":"Create a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"Best Practices for Remote Team Collaboration\" with the question description be \"What strategies help remote teams stay productive and connected?\". The answer should be \"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout.\"\n```","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Best Practices for Remote Team Collaboration","article_published":"Online","question_desc":"What strategies help remote teams stay productive and connected?","answer_desc":"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Best Practices for Remote Team Collaboration","question_desc":"What strategies help remote teams stay productive and connected?","answer_desc":"Effective communication using video calls and instant messaging is key. Setting clear goals and deadlines keeps teams aligned. Regular check-ins foster accountability and morale. Using shared digital tools ensures everyone has access to the latest information. Prioritizing work-life balance prevents burnout."},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/282"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L506","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_knowledge_article","data_retrieval_spec":[{"type":"soql","id":"knowledge_article_info","params":{"query":"SELECT Id, PublishStatus, Question__c, FAQ_Answer__c FROM Knowledge__kav WHERE Title='{article_title}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"4f5665ecbd2bada61ce3d7bfca7c14ef817bf66e8008211e1bcc67258ca65712"} {"id":"scuba:service_010_004","source_id":"scuba","original_id":"service_010_004","kind":"benchmark_task","instruction":"Create a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_004","query_templated":"Create a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"","query":"Create a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"How to Reset Your Password","article_published":"Online","question_desc":"How can I reset my password if I forgot it?","answer_desc":"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"How to Reset Your Password","question_desc":"How can I reset my password if I forgot it?","answer_desc":"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/283","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Knowledge Article","source_difficulty":"easy","template_id":"service_010","template_name":"create_knowledge_article","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/283"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/283"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_knowledge_article"}],"source_schema_notes":[],"template_registry":{"name":"create_knowledge_article","original":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be '{article_title}' and the article detail should be '{article_content}'.","prerequisites":{},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article","differs_from_embedded_fields":["objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_004","query_templated":"Create a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"How to Reset Your Password\" with the question description be \"How can I reset my password if I forgot it?\". The answer should be \"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes.\"\n```","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"How to Reset Your Password","article_published":"Online","question_desc":"How can I reset my password if I forgot it?","answer_desc":"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"How to Reset Your Password","question_desc":"How can I reset my password if I forgot it?","answer_desc":"To reset your password, go to the login page and click 'Forgot Password'. Enter your email address and follow the instructions sent to your email. The reset link expires in 24 hours for security purposes."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/283"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L506","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_knowledge_article","data_retrieval_spec":[{"type":"soql","id":"knowledge_article_info","params":{"query":"SELECT Id, PublishStatus, Question__c, FAQ_Answer__c FROM Knowledge__kav WHERE Title='{article_title}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"fc43b3295ab027c7ac0d957dac723b6df417e79f21f478aeccd8696e47e7e8ea"} {"id":"scuba:service_010_005","source_id":"scuba","original_id":"service_010_005","kind":"benchmark_task","instruction":"Create a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_005","query_templated":"Create a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"","query":"Create a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Troubleshooting Network Connectivity Issues","article_published":"Online","question_desc":"What should I do if I'm experiencing network connectivity problems?","answer_desc":"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Troubleshooting Network Connectivity Issues","question_desc":"What should I do if I'm experiencing network connectivity problems?","answer_desc":"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/284","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Knowledge Article","source_difficulty":"easy","template_id":"service_010","template_name":"create_knowledge_article","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/284"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/284"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_knowledge_article"}],"source_schema_notes":[],"template_registry":{"name":"create_knowledge_article","original":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be '{article_title}' and the article detail should be '{article_content}'.","prerequisites":{},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_knowledge_article","differs_from_embedded_fields":["objects","prerequisites","template_string"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Knowledge Article","task_id":"service_010_005","query_templated":"Create a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"","query":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again.\n\n## Your ultimate task\n```\nCreate a knowledge article and publish it. The title should be \"Troubleshooting Network Connectivity Issues\" with the question description be \"What should I do if I'm experiencing network connectivity problems?\". The answer should be \"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see.\"\n```","human_review_comment":"reject; use original","ground_truth_dict":{"article_title":"Troubleshooting Network Connectivity Issues","article_published":"Online","question_desc":"What should I do if I'm experiencing network connectivity problems?","answer_desc":"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see."},"difficulty":"easy","query_template_name":"create_knowledge_article","query_template_metadata":{"template_id":"service_010","template_string":"Create a knowledge article and publish it. The title should be {article_title} with the question description be \"{question_desc}\". The answer should be \"{answer_desc}\".","prerequisites":{"metadata":{"types_and_members":{"CustomObject":["Knowledge__kav"]}}},"objects":["Knowledge__ka"],"metadata_types":[]},"instance_dict":{"article_title":"Troubleshooting Network Connectivity Issues","question_desc":"What should I do if I'm experiencing network connectivity problems?","answer_desc":"If you're experiencing network connectivity issues, first check your cable connections. Restart your router and modem. If the problem persists, contact your internet service provider. Document any error messages you see."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:Create a knowledge article and publish it. The title should be “Headlight Replacement” with the question description \"How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?\". The answer should be \"To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “Knowledge,” and open the Knowledge app.\n- Click New to create a new article.\n- Enter Title: “Headlight Replacement”.\n- Enter Question: “How can I safely replace a malfunctioning car headlight on my own, including steps for removal, installation, and precautions to avoid damage?”\n- Scroll to the Answer section and enter:\n “To replace a malfunctioning car headlight, first park your vehicle on level ground, turn off the ignition, and open the hood. Locate the headlight housing behind the light assembly. Disconnect the power connector by gently pulling it off, then remove the dust cover if present. Release the bulb by undoing the retaining clip or turning it counterclockwise. Avoid touching the glass of the new bulb with bare fingers, as oils may shorten its lifespan; hold it by the base instead. Insert the new bulb securely, reattach the clip and cover, then reconnect the wiring. Finally, turn on the headlights to confirm proper operation.”\n- Click Save.\n- Click Publish, then confirm Publish in the dialog.\nSample 2:\nTASK:Create a knowledge article and publish it. The title should be \"The Science Behind Sleep and Productivity\" with the question description \"How does sleep quality impact workplace productivity and cognitive function?\". The answer should be \"Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.\"\nEXPERIENCE:\nSuccessful plan to create and publish the knowledge article:\n\n- Log in to the system.\n- Open App Launcher, search for “knowledge,” and select “Knowledge.”\n- Click “New” to create a new article.\n- Enter Title: “The Science Behind Sleep and Productivity.”\n- Enter Question: “How does sleep quality impact workplace productivity and cognitive function?”\n- Scroll to the Answer field if needed and click into it.\n- Enter Answer: “Research shows that quality sleep improves memory, decision-making, and creativity. Lack of sleep reduces focus and increases errors. Understanding sleep cycles can help optimize work schedules. Employers can support better sleep habits to boost employee performance and wellbeing.”\n- Click “Save.”\n- Click “Publish,” then confirm by clicking “Publish” again."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/284"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L506","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_knowledge_article","data_retrieval_spec":[{"type":"soql","id":"knowledge_article_info","params":{"query":"SELECT Id, PublishStatus, Question__c, FAQ_Answer__c FROM Knowledge__kav WHERE Title='{article_title}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"b15f647b69e9233bb52abd2353ca40622192ef36f9604f190619d25139fa411b"} {"id":"scuba:service_011_001","source_id":"scuba","original_id":"service_011_001","kind":"benchmark_task","instruction":"A customer just called saying he can’t get an order through for 35 units of CollabDesign Studio. Using the knowledge article, write a brief, customer-ready note that explains what’s preventing the order. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"qa","task_id":"service_011_001","query_templated":"A customer called in to complain that he cannot place an order on CollabDesign Studio with 35 units.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"A customer just called saying he can’t get an order through for 35 units of CollabDesign Studio. Using the knowledge article, write a brief, customer-ready note that explains what’s preventing the order. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","human_review_comment":"accept","ground_truth_dict":{"answer":"Each order is limited to 25 units."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to complain that he cannot place an order on CollabDesign Studio with 35 units."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/285","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"qa","source_difficulty":"medium","template_id":"service_011","template_name":"qa_service_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/285"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/285"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_service_001","original":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer just called saying he can’t get an order through for 35 units of CollabDesign Studio. Using the knowledge article, write a brief, customer-ready note that explains what’s preventing the order. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"service","subcategory":"qa","task_id":"service_011_001","query_templated":"A customer called in to complain that he cannot place an order on CollabDesign Studio with 35 units.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer just called saying he can’t get an order through for 35 units of CollabDesign Studio. Using the knowledge article, write a brief, customer-ready note that explains what’s preventing the order. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Each order is limited to 25 units."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to complain that he cannot place an order on CollabDesign Studio with 35 units."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/285"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"8f27d1b77a4de59e69b84c62f27084bcfc13afefa55d1399130cd9fe9c2df6a9"} {"id":"scuba:service_011_002","source_id":"scuba","original_id":"service_011_002","kind":"benchmark_task","instruction":"While trying to buy AIOptics Vision, the customer saw that Workflow Genius and AI DesignShift were automatically added at checkout and got confused. Please use the knowledge article to provide a short, clear explanation for why those items are auto-added. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"qa","task_id":"service_011_002","query_templated":"A customer wants to place an order on the AIOptics Vision product. However, at the checkout page, the customer realized that Workflow Genius and AI DesignShift are automatically added to the order. The customer is confused and wants to know why.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"While trying to buy AIOptics Vision, the customer saw that Workflow Genius and AI DesignShift were automatically added at checkout and got confused. Please use the knowledge article to provide a short, clear explanation for why those items are auto-added. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","human_review_comment":"accept","ground_truth_dict":{"answer":"AIOptics Vision requires the Workflow Genius and AI DesignShift as part of its package."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer wants to place an order on the AIOptics Vision product. However, at the checkout page, the customer realized that Workflow Genius and AI DesignShift are automatically added to the order. The customer is confused and wants to know why."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/286","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"qa","source_difficulty":"medium","template_id":"service_011","template_name":"qa_service_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/286"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/286"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_service_001","original":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nWhile trying to buy AIOptics Vision, the customer saw that Workflow Genius and AI DesignShift were automatically added at checkout and got confused. Please use the knowledge article to provide a short, clear explanation for why those items are auto-added. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"service","subcategory":"qa","task_id":"service_011_002","query_templated":"A customer wants to place an order on the AIOptics Vision product. However, at the checkout page, the customer realized that Workflow Genius and AI DesignShift are automatically added to the order. The customer is confused and wants to know why.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nWhile trying to buy AIOptics Vision, the customer saw that Workflow Genius and AI DesignShift were automatically added at checkout and got confused. Please use the knowledge article to provide a short, clear explanation for why those items are auto-added. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"AIOptics Vision requires the Workflow Genius and AI DesignShift as part of its package."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer wants to place an order on the AIOptics Vision product. However, at the checkout page, the customer realized that Workflow Genius and AI DesignShift are automatically added to the order. The customer is confused and wants to know why."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/286"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"4d8065c50209e7ab29ca7b2c737e05e01974d66a5a9a49f4b9c813bd80b7f382"} {"id":"scuba:service_011_003","source_id":"scuba","original_id":"service_011_003","kind":"benchmark_task","instruction":"A Sales Rep tried to apply a 15% discount to a quote totaling $18, but the manager rejected it. Reference the knowledge article and compose a concise reply that explains the reason for the rejection. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"qa","task_id":"service_011_003","query_templated":"The Sales Rep wants to apply 15% discount to the quote whose purchase is $18. However, the quote was rejected by his manager. The Sales Rep is confused and wants to know why.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"A Sales Rep tried to apply a 15% discount to a quote totaling $18, but the manager rejected it. Reference the knowledge article and compose a concise reply that explains the reason for the rejection. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","human_review_comment":"accept","ground_truth_dict":{"answer":"15% Discount is for Purchases Over $20."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"The Sales Rep wants to apply 15% discount to the quote whose purchase is $18. However, the quote was rejected by his manager. The Sales Rep is confused and wants to know why."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/287","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"qa","source_difficulty":"medium","template_id":"service_011","template_name":"qa_service_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/287"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/287"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_service_001","original":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA Sales Rep tried to apply a 15% discount to a quote totaling $18, but the manager rejected it. Reference the knowledge article and compose a concise reply that explains the reason for the rejection. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"service","subcategory":"qa","task_id":"service_011_003","query_templated":"The Sales Rep wants to apply 15% discount to the quote whose purchase is $18. However, the quote was rejected by his manager. The Sales Rep is confused and wants to know why.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA Sales Rep tried to apply a 15% discount to a quote totaling $18, but the manager rejected it. Reference the knowledge article and compose a concise reply that explains the reason for the rejection. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"15% Discount is for Purchases Over $20."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"The Sales Rep wants to apply 15% discount to the quote whose purchase is $18. However, the quote was rejected by his manager. The Sales Rep is confused and wants to know why."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/287"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"1b7e7179c4b4101005cedcc5fd8ff040a9dfd5ff06ec1471b31a57fd08b56224"} {"id":"scuba:service_011_004","source_id":"scuba","original_id":"service_011_004","kind":"benchmark_task","instruction":"A customer is upset because they can’t request a refund on an order placed 15 days ago. Based on the knowledge article, draft a short message clarifying why a refund isn’t available now. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"qa","task_id":"service_011_004","query_templated":"A customer called in to complain that he cannot request a refund on an order placed 15 days ago.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"A customer is upset because they can’t request a refund on an order placed 15 days ago. Based on the knowledge article, draft a short message clarifying why a refund isn’t available now. Note: The knowledge article does exist, but you may need to switch between different list views to find it.","human_review_comment":"accept","ground_truth_dict":{"answer":"Refunds are only available for orders placed within the last 7 days."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to complain that he cannot request a refund on an order placed 15 days ago."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/288","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"qa","source_difficulty":"medium","template_id":"service_011","template_name":"qa_service_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/288"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/288"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_service_001","original":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer is upset because they can’t request a refund on an order placed 15 days ago. Based on the knowledge article, draft a short message clarifying why a refund isn’t available now. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"service","subcategory":"qa","task_id":"service_011_004","query_templated":"A customer called in to complain that he cannot request a refund on an order placed 15 days ago.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer is upset because they can’t request a refund on an order placed 15 days ago. Based on the knowledge article, draft a short message clarifying why a refund isn’t available now. Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","human_review_comment":"accept","ground_truth_dict":{"answer":"Refunds are only available for orders placed within the last 7 days."},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to complain that he cannot request a refund on an order placed 15 days ago."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/288"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"1181934faf5cf4cf5f344a29c3e5fb7629e588e0bb6647924ff6d0697978d6e9"} {"id":"scuba:service_011_005","source_id":"scuba","original_id":"service_011_005","kind":"benchmark_task","instruction":"A customer asked for an additional discount, saying they have a competing offer from another vendor. Per the knowledge article, send a brief reply with only \"Yes\" or \"No\". Note: The knowledge article does exist, but you may need to switch between different list views to find it.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"qa","task_id":"service_011_005","query_templated":"A customer called in to request additional discount since he has a competing offer from another vendor. The customer wants to know if the discount can be applied.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"A customer asked for an additional discount, saying they have a competing offer from another vendor. Per the knowledge article, send a brief reply with only \"Yes\" or \"No\". Note: The knowledge article does exist, but you may need to switch between different list views to find it.","human_review_comment":"modifed to align with GT","ground_truth_dict":{"answer":"Yes"},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to request additional discount since he has a competing offer from another vendor. The customer wants to know if the discount can be applied."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/289","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"qa","source_difficulty":"medium","template_id":"service_011","template_name":"qa_service_001","evaluation_mode":"question_answering","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/289"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/289"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001"}],"source_schema_notes":[],"template_registry":{"name":"qa_service_001","original":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_qa.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_qa.json","sha256":"512e7cfb68a13ea0bc1575ff5435f658339f1095b54b29aa573a0b1ec7146828","source_json_pointer":"/qa_service_001","differs_from_embedded_fields":[]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer asked for an additional discount, saying they have a competing offer from another vendor. Per the knowledge article, send a brief reply with only \"Yes\" or \"No\". Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","memory":"## Similar task and experience\n","memory_content_status":"heading_only","original":{"category":"service","subcategory":"qa","task_id":"service_011_005","query_templated":"A customer called in to request additional discount since he has a competing offer from another vendor. The customer wants to know if the discount can be applied.. Grounded on the knowledge article, draft a short reply to explain the reason.","query":"## Similar task and experience\n\n\n## Your ultimate task\n```\nA customer asked for an additional discount, saying they have a competing offer from another vendor. Per the knowledge article, send a brief reply with only \"Yes\" or \"No\". Note: The knowledge article does exist, but you may need to switch between different list views to find it.\n```","human_review_comment":"modifed to align with GT","ground_truth_dict":{"answer":"Yes"},"difficulty":"medium","query_template_name":"qa_service_001","query_template_metadata":{"template_id":"service_011","template_string":"{one_line_issue}. Grounded on the knowledge article, draft a short reply to explain the reason.","prerequisites":{"metadata":{"types_and_members":{"Settings":["Knowledge"],"CustomObject":["Knowledge__kav","VoiceCallTranscript__c"],"Layout":["VoiceCallTranscript__c-VoiceCallTranscript Layout"],"Profile":["Admin"]}},"data":{"objects":["Knowledge__kav"]}},"objects":["Knowledge__kav"],"metadata_types":[]},"instance_dict":{"one_line_issue":"A customer called in to request additional discount since he has a competing offer from another vendor. The customer wants to know if the discount can be applied."},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n"},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/289"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L76","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L63","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/helpers/utils.py#L62"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"literal_gate_then_normalized_token_f1","method":"MilestoneEvaluator.qa_evaluation","scoring_scope":"Literal ground-truth inclusion gate, then normalized token F1; not a semantic model judge."},"instruction_fingerprint":"c40f5ad7df7bd1bfd215d1a8a68ff582d7996c0d74799f193e845db790b9780a"} {"id":"scuba:service_013_001","source_id":"scuba","original_id":"service_013_001","kind":"benchmark_task","instruction":"Can you open an incident for widespread battery drain on our portable chargers? Multiple customers are seeing the PowerCell X1 and EnergyBoost Max losing charge quickly, disrupting daily operations. Note: PowerCell X1 is partially business blocking, while EnergyBoost Max isn’t blocking since some functions still work. Use these fields—Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_001","query_templated":"Multiple customers have reported battery drain issues with the PowerCell X1 and EnergyBoost Max portable chargers, affecting their daily operations. This is causing partially business blocking impact to product PowerCell X1 but no blocking impact to EnergyBoost Max as some functions remain operational. Create an incident record with the following details: Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.","query":"Can you open an incident for widespread battery drain on our portable chargers? Multiple customers are seeing the PowerCell X1 and EnergyBoost Max losing charge quickly, disrupting daily operations. Note: PowerCell X1 is partially business blocking, while EnergyBoost Max isn’t blocking since some functions still work. Use these fields—Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"battery performance degradation in portable charging devices","urgency":"Medium","impact":"Low","priority":"Moderate","products":["PowerCell X1","EnergyBoost Max"],"product_impact_type":["Partially Business-Blocking","Not Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"Multiple customers have reported battery drain issues with the PowerCell X1 and EnergyBoost Max portable chargers, affecting their daily operations. This is causing partially business blocking impact to product PowerCell X1 but no blocking impact to EnergyBoost Max as some functions remain operational","incident_details":"Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/290","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident Management","source_difficulty":"medium","template_id":"service_013","template_name":"create_incident_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/290"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/290"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_incident_record"}],"source_schema_notes":[],"template_registry":{"name":"create_incident_record","original":{"template_id":"service_012","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["products.json"]}}},"objects":["Incident"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record","differs_from_embedded_fields":["prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCan you open an incident for widespread battery drain on our portable chargers? Multiple customers are seeing the PowerCell X1 and EnergyBoost Max losing charge quickly, disrupting daily operations. Note: PowerCell X1 is partially business blocking, while EnergyBoost Max isn’t blocking since some functions still work. Use these fields—Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_001","query_templated":"Multiple customers have reported battery drain issues with the PowerCell X1 and EnergyBoost Max portable chargers, affecting their daily operations. This is causing partially business blocking impact to product PowerCell X1 but no blocking impact to EnergyBoost Max as some functions remain operational. Create an incident record with the following details: Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.","query":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nCan you open an incident for widespread battery drain on our portable chargers? Multiple customers are seeing the PowerCell X1 and EnergyBoost Max losing charge quickly, disrupting daily operations. Note: PowerCell X1 is partially business blocking, while EnergyBoost Max isn’t blocking since some functions still work. Use these fields—Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium.\n```","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"battery performance degradation in portable charging devices","urgency":"Medium","impact":"Low","priority":"Moderate","products":["PowerCell X1","EnergyBoost Max"],"product_impact_type":["Partially Business-Blocking","Not Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"Multiple customers have reported battery drain issues with the PowerCell X1 and EnergyBoost Max portable chargers, affecting their daily operations. This is causing partially business blocking impact to product PowerCell X1 but no blocking impact to EnergyBoost Max as some functions remain operational","incident_details":"Subject: battery performance degradation in portable charging devices; Urgency: medium; Impact: low; Priority: medium"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/290"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L576","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_incident_record","data_retrieval_spec":[{"type":"soql","id":"incident_record","params":{"query":"SELECT Id FROM Incident WHERE Subject='{incident_subject}' AND Urgency='{urgency}' AND Impact='{impact}' AND Priority='{priority}'"}},{"type":"soql","id":"incident_product_relationship","params":{"query":"SELECT Product2Id, Product2.Name, ImpactType FROM IncidentRelatedItem WHERE IncidentId='{retrieved_results['incident_record'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"c1910a9f3e30b79b856a6fe54f7e379765cfa3b8c4fd519ba053ee6aade6ebf2"} {"id":"scuba:service_013_002","source_id":"scuba","original_id":"service_013_002","kind":"benchmark_task","instruction":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_002","query_templated":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.","query":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"data corruption in storage systems","urgency":"High","impact":"High","priority":"High","products":["StorageMax Pro"],"product_impact_type":["Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations","incident_details":"Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/291","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident Management","source_difficulty":"medium","template_id":"service_013","template_name":"create_incident_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/291"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/291"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_incident_record"}],"source_schema_notes":[],"template_registry":{"name":"create_incident_record","original":{"template_id":"service_012","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["products.json"]}}},"objects":["Incident"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record","differs_from_embedded_fields":["prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nThere are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_002","query_templated":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.","query":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nThere are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"data corruption in storage systems","urgency":"High","impact":"High","priority":"High","products":["StorageMax Pro"],"product_impact_type":["Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations","incident_details":"Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/291"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L576","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_incident_record","data_retrieval_spec":[{"type":"soql","id":"incident_record","params":{"query":"SELECT Id FROM Incident WHERE Subject='{incident_subject}' AND Urgency='{urgency}' AND Impact='{impact}' AND Priority='{priority}'"}},{"type":"soql","id":"incident_product_relationship","params":{"query":"SELECT Product2Id, Product2.Name, ImpactType FROM IncidentRelatedItem WHERE IncidentId='{retrieved_results['incident_record'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"2addbaae5bd817842f153d514566f1254829a61b96f92b294d030d278bba1110"} {"id":"scuba:service_013_003","source_id":"scuba","original_id":"service_013_003","kind":"benchmark_task","instruction":"Heads up—there are 5 new cases about users failing to log in on SecureAuth Pro and AccessGuard, our authentication products. Customers say it’s partially business blocking for their security workflows. Create an incident with these details—Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_003","query_templated":"There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.","query":"Heads up—there are 5 new cases about users failing to log in on SecureAuth Pro and AccessGuard, our authentication products. Customers say it’s partially business blocking for their security workflows. Create an incident with these details—Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"authentication failures in security systems","urgency":"Medium","impact":"Medium","priority":"Moderate","products":["SecureAuth Pro","AccessGuard"],"product_impact_type":["Partially Business-Blocking","Partially Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations","incident_details":"Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/292","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident Management","source_difficulty":"medium","template_id":"service_013","template_name":"create_incident_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/292"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/292"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_incident_record"}],"source_schema_notes":[],"template_registry":{"name":"create_incident_record","original":{"template_id":"service_012","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["products.json"]}}},"objects":["Incident"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record","differs_from_embedded_fields":["prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nHeads up—there are 5 new cases about users failing to log in on SecureAuth Pro and AccessGuard, our authentication products. Customers say it’s partially business blocking for their security workflows. Create an incident with these details—Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_003","query_templated":"There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.","query":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nHeads up—there are 5 new cases about users failing to log in on SecureAuth Pro and AccessGuard, our authentication products. Customers say it’s partially business blocking for their security workflows. Create an incident with these details—Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium.\n```","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"authentication failures in security systems","urgency":"Medium","impact":"Medium","priority":"Moderate","products":["SecureAuth Pro","AccessGuard"],"product_impact_type":["Partially Business-Blocking","Partially Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations","incident_details":"Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: medium"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/292"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L576","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_incident_record","data_retrieval_spec":[{"type":"soql","id":"incident_record","params":{"query":"SELECT Id FROM Incident WHERE Subject='{incident_subject}' AND Urgency='{urgency}' AND Impact='{impact}' AND Priority='{priority}'"}},{"type":"soql","id":"incident_product_relationship","params":{"query":"SELECT Product2Id, Product2.Name, ImpactType FROM IncidentRelatedItem WHERE IncidentId='{retrieved_results['incident_record'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"bc71f08d408283b4bba45fd14ea3eb9224c9fa10d11e2a7b73403a6a7b102103"} {"id":"scuba:service_013_004","source_id":"scuba","original_id":"service_013_004","kind":"benchmark_task","instruction":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_004","query_templated":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.","query":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"performance degradation in workflow systems","urgency":"Low","impact":"Low","priority":"Low","products":["DataFlow X2","ProcessEngine Max"],"product_impact_type":["Not Business-Blocking","Not Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available","incident_details":"Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/293","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident Management","source_difficulty":"medium","template_id":"service_013","template_name":"create_incident_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/293"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/293"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_incident_record"}],"source_schema_notes":[],"template_registry":{"name":"create_incident_record","original":{"template_id":"service_012","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["products.json"]}}},"objects":["Incident"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record","differs_from_embedded_fields":["prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nMultiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_004","query_templated":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.","query":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nMultiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available. Create an incident record with the following details: Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"performance degradation in workflow systems","urgency":"Low","impact":"Low","priority":"Low","products":["DataFlow X2","ProcessEngine Max"],"product_impact_type":["Not Business-Blocking","Not Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident"],"metadata_types":[]},"instance_dict":{"incident_context":"Multiple customers have reported slow response times with the DataFlow X2 and ProcessEngine Max workflow systems, affecting their operational efficiency. This is causing not business blocking impact to both products as workarounds are available","incident_details":"Subject: performance degradation in workflow systems; Urgency: low; Impact: low; Priority: low"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/293"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L576","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_incident_record","data_retrieval_spec":[{"type":"soql","id":"incident_record","params":{"query":"SELECT Id FROM Incident WHERE Subject='{incident_subject}' AND Urgency='{urgency}' AND Impact='{impact}' AND Priority='{priority}'"}},{"type":"soql","id":"incident_product_relationship","params":{"query":"SELECT Product2Id, Product2.Name, ImpactType FROM IncidentRelatedItem WHERE IncidentId='{retrieved_results['incident_record'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"54332bce4973113591253b7ba892130a94302198522bc03f7e509ca4e1e0847a"} {"id":"scuba:service_013_005","source_id":"scuba","original_id":"service_013_005","kind":"benchmark_task","instruction":"Urgent request: there are 8 new cases of overheating on the ThermalMax Pro and CoolBreeze Elite systems—our latest HVAC line—and it’s blocking facility operations for customers. Raise an incident with—Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_005","query_templated":"There are 8 new cases complaining about overheating issues with the ThermalMax Pro and CoolBreeze Elite cooling systems, our latest HVAC products. This is causing business blocking impact for customers' facility operations. Create an incident record with the following details: Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.","query":"Urgent request: there are 8 new cases of overheating on the ThermalMax Pro and CoolBreeze Elite systems—our latest HVAC line—and it’s blocking facility operations for customers. Raise an incident with—Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"overheating defects in HVAC cooling systems","urgency":"High","impact":"High","priority":"High","products":["ThermalMax Pro","CoolBreeze Elite"],"product_impact_type":["Business-Blocking","Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident","IncidentRelatedItem"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 8 new cases complaining about overheating issues with the ThermalMax Pro and CoolBreeze Elite cooling systems, our latest HVAC products. This is causing business blocking impact for customers' facility operations","incident_details":"Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/294","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident Management","source_difficulty":"medium","template_id":"service_013","template_name":"create_incident_record","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/294"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/294"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_incident_record"}],"source_schema_notes":[],"template_registry":{"name":"create_incident_record","original":{"template_id":"service_012","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["products.json"]}}},"objects":["Incident"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_incident_record","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nUrgent request: there are 8 new cases of overheating on the ThermalMax Pro and CoolBreeze Elite systems—our latest HVAC line—and it’s blocking facility operations for customers. Raise an incident with—Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident Management","task_id":"service_013_005","query_templated":"There are 8 new cases complaining about overheating issues with the ThermalMax Pro and CoolBreeze Elite cooling systems, our latest HVAC products. This is causing business blocking impact for customers' facility operations. Create an incident record with the following details: Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.","query":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used.\n\n## Your ultimate task\n```\nUrgent request: there are 8 new cases of overheating on the ThermalMax Pro and CoolBreeze Elite systems—our latest HVAC line—and it’s blocking facility operations for customers. Raise an incident with—Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high.\n```","human_review_comment":"accept","ground_truth_dict":{"incident_subject":"overheating defects in HVAC cooling systems","urgency":"High","impact":"High","priority":"High","products":["ThermalMax Pro","CoolBreeze Elite"],"product_impact_type":["Business-Blocking","Business-Blocking"]},"difficulty":"medium","query_template_name":"create_incident_record","query_template_metadata":{"template_id":"service_013","template_string":"{incident_context}. Create an incident record with the following details: {incident_details}.","prerequisites":{"data":{"objects":{"Product2":["Product2.json"]}}},"objects":["Incident","IncidentRelatedItem"],"metadata_types":[]},"instance_dict":{"incident_context":"There are 8 new cases complaining about overheating issues with the ThermalMax Pro and CoolBreeze Elite cooling systems, our latest HVAC products. This is causing business blocking impact for customers' facility operations","incident_details":"Subject: overheating defects in HVAC cooling systems; Urgency: high; Impact: high; Priority: high"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:There are 12 new cases complaining about data corruption issues with the StorageMax Pro system, our primary data storage product. This is causing business blocking impact for customers' data operations. Create an incident record with the following details: Subject: data corruption in storage systems; Urgency: high; Impact: high; Priority: high.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to start a new incident.\n- Fill in the fields:\n - Subject: Data corruption in storage systems\n - Description: 12 new cases have been reported regarding data corruption issues in the StorageMax Pro system, our primary data storage product. This is causing business-blocking impact for customers' data operations.\n - Priority: High (open the Priority dropdown and select High)\n- Click “Save” to create the incident.\n\nNotes:\n- No hotkeys were used in the trajectory.\nSample 2:\nTASK:There are 5 new cases complaining about login failures with the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business blocking impact for customers' security operations. Create an incident record with the following details: Subject: authentication failures in security systems; Urgency: medium; Impact: medium; Priority: moderate.\nEXPERIENCE:\nSuccessful plan to create the incident:\n\n- Log in to the system.\n- Open App Launcher, search for “incidents,” and select “Incidents.”\n- Click “New” to create a new incident.\n- Fill fields:\n - Subject: “Authentication failures in security systems”\n - Description: “5 new cases have been reported regarding login failures in the SecureAuth Pro and AccessGuard systems, our authentication products. This is causing partially business-blocking impact for customers' security operations.”\n - Set Urgency to Medium (via dropdown).\n - Set Impact to Medium (via dropdown).\n - Set Priority to Moderate (via dropdown).\n- Click “Save.”\n\nNotes:\n- No hotkeys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/294"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L576","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_incident_record","data_retrieval_spec":[{"type":"soql","id":"incident_record","params":{"query":"SELECT Id FROM Incident WHERE Subject='{incident_subject}' AND Urgency='{urgency}' AND Impact='{impact}' AND Priority='{priority}'"}},{"type":"soql","id":"incident_product_relationship","params":{"query":"SELECT Product2Id, Product2.Name, ImpactType FROM IncidentRelatedItem WHERE IncidentId='{retrieved_results['incident_record'].records[0]['Id']}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"89d2b69414a95575bdee82013422ed0dfe1997dc3a82bf8900fefc1caaae69b8"} {"id":"scuba:service_014_001","source_id":"scuba","original_id":"service_014_001","kind":"benchmark_task","instruction":"After an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_001","query_templated":"After an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.","query":"After an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"authentication service outage affecting enterprise applications","change_request_subject":"upgrade authentication infrastructure","change_request_priority":"High","change_request_impact":"Medium","change_request_risk_level":"Low","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"authentication service outage affecting enterprise applications","change_request_details":"Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/295","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident management","source_difficulty":"medium","template_id":"service_014","template_name":"create_change_request_link_incident","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/295"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/295"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_change_request_link_incident"}],"source_schema_notes":[],"template_registry":{"name":"create_change_request_link_incident","original":{"template_id":"service_013","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["incidents.json"]}}},"objects":["ChangeRequest"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nAfter an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_001","query_templated":"After an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.","query":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nAfter an intensive investigation, the root cause for the incident 'authentication service outage affecting enterprise applications' is found. We need to create a change request with the following details. Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to in progress.\n```","human_review_comment":"reject; use original","ground_truth_dict":{"incident_subject":"authentication service outage affecting enterprise applications","change_request_subject":"upgrade authentication infrastructure","change_request_priority":"High","change_request_impact":"Medium","change_request_risk_level":"Low","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"authentication service outage affecting enterprise applications","change_request_details":"Subject: upgrade authentication infrastructure, Description: implement redundant authentication servers to prevent future service outages, Priority: High, Impact: Medium, Risk Level: Low","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/295"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L605","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_change_request_link_incident","data_retrieval_spec":[{"type":"soql","id":"change_request_info","params":{"query":"SELECT Id, ChangeRequestNumber, Impact, Priority, RiskLevel FROM ChangeRequest WHERE Subject='{change_request_subject}'"}},{"type":"soql","id":"incident_id_info","params":{"query":"SELECT RelatedIssueId FROM ChangeRequestRelatedIssue WHERE ChangeRequestId='{retrieved_results['change_request_info'].records[0]['Id']}' AND RelatedEntityType='Incident'"}},{"type":"soql","id":"incident_info","params":{"query":"SELECT Id, Subject, Status FROM Incident WHERE Subject='{incident_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"980e8fe3e7e79a6897756cfa98070cd52161242f6a119a5d8f1040a9f4274ce6"} {"id":"scuba:service_014_002","source_id":"scuba","original_id":"service_014_002","kind":"benchmark_task","instruction":"The root cause for the incident “data sync service disruption across mobile and web platforms” is confirmed. Go ahead and raise a change request titled “implement data synchronization monitoring” with this description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies. Set Priority to Medium, Impact to High, and Risk Level to Medium. Associate the CR with the incident ticket and set the incident’s status to open.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_002","query_templated":"After an intensive investigation, the root cause for the incident 'data sync service disruption across mobile and web platforms' is found. We need to create a change request with the following details. Subject: implement data synchronization monitoring, Description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies, Priority: Medium, Impact: High, Risk Level: Medium. Link this change request to the incident case and update the incident status to open.","query":"The root cause for the incident “data sync service disruption across mobile and web platforms” is confirmed. Go ahead and raise a change request titled “implement data synchronization monitoring” with this description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies. Set Priority to Medium, Impact to High, and Risk Level to Medium. Associate the CR with the incident ticket and set the incident’s status to open.","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"data sync service disruption across mobile and web platforms","change_request_subject":"implement data synchronization monitoring","change_request_priority":"Moderate","change_request_impact":"High","change_request_risk_level":"Medium","incident_new_status":"Open"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"data sync service disruption across mobile and web platforms","change_request_details":"Subject: implement data synchronization monitoring, Description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies, Priority: Medium, Impact: High, Risk Level: Medium","new_status":"open"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/296","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident management","source_difficulty":"medium","template_id":"service_014","template_name":"create_change_request_link_incident","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/296"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/296"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_change_request_link_incident"}],"source_schema_notes":[],"template_registry":{"name":"create_change_request_link_incident","original":{"template_id":"service_013","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["incidents.json"]}}},"objects":["ChangeRequest"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe root cause for the incident “data sync service disruption across mobile and web platforms” is confirmed. Go ahead and raise a change request titled “implement data synchronization monitoring” with this description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies. Set Priority to Medium, Impact to High, and Risk Level to Medium. Associate the CR with the incident ticket and set the incident’s status to open.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_002","query_templated":"After an intensive investigation, the root cause for the incident 'data sync service disruption across mobile and web platforms' is found. We need to create a change request with the following details. Subject: implement data synchronization monitoring, Description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies, Priority: Medium, Impact: High, Risk Level: Medium. Link this change request to the incident case and update the incident status to open.","query":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nThe root cause for the incident “data sync service disruption across mobile and web platforms” is confirmed. Go ahead and raise a change request titled “implement data synchronization monitoring” with this description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies. Set Priority to Medium, Impact to High, and Risk Level to Medium. Associate the CR with the incident ticket and set the incident’s status to open.\n```","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"data sync service disruption across mobile and web platforms","change_request_subject":"implement data synchronization monitoring","change_request_priority":"Moderate","change_request_impact":"High","change_request_risk_level":"Medium","incident_new_status":"Open"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"data sync service disruption across mobile and web platforms","change_request_details":"Subject: implement data synchronization monitoring, Description: deploy real-time sync monitoring tools to detect and prevent data inconsistencies, Priority: Medium, Impact: High, Risk Level: Medium","new_status":"open"},"on_start":null,"on_finish":null,"in_domain":false,"has_annotation":false,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/296"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L605","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_change_request_link_incident","data_retrieval_spec":[{"type":"soql","id":"change_request_info","params":{"query":"SELECT Id, ChangeRequestNumber, Impact, Priority, RiskLevel FROM ChangeRequest WHERE Subject='{change_request_subject}'"}},{"type":"soql","id":"incident_id_info","params":{"query":"SELECT RelatedIssueId FROM ChangeRequestRelatedIssue WHERE ChangeRequestId='{retrieved_results['change_request_info'].records[0]['Id']}' AND RelatedEntityType='Incident'"}},{"type":"soql","id":"incident_info","params":{"query":"SELECT Id, Subject, Status FROM Incident WHERE Subject='{incident_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"34fe6bf3106ac1aeebcc091292c3a0953634f25327fe0fec935efc48026727f7"} {"id":"scuba:service_014_003","source_id":"scuba","original_id":"service_014_003","kind":"benchmark_task","instruction":"We’ve identified what caused the “network connectivity failures across regional offices” incident. Create a change request now: Subject—upgrade network infrastructure; Description—implement redundant network connections to prevent future connectivity issues; Priority—High; Impact—Low; Risk Level—Medium. Link the CR back to the incident and update the incident status to in progress.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_003","query_templated":"After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.","query":"We’ve identified what caused the “network connectivity failures across regional offices” incident. Create a change request now: Subject—upgrade network infrastructure; Description—implement redundant network connections to prevent future connectivity issues; Priority—High; Impact—Low; Risk Level—Medium. Link the CR back to the incident and update the incident status to in progress.","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"network connectivity failures across regional offices","change_request_subject":"upgrade network infrastructure","change_request_priority":"High","change_request_impact":"Low","change_request_risk_level":"Medium","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"network connectivity failures across regional offices","change_request_details":"Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/297","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident management","source_difficulty":"medium","template_id":"service_014","template_name":"create_change_request_link_incident","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/297"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/297"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_change_request_link_incident"}],"source_schema_notes":[],"template_registry":{"name":"create_change_request_link_incident","original":{"template_id":"service_013","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["incidents.json"]}}},"objects":["ChangeRequest"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nWe’ve identified what caused the “network connectivity failures across regional offices” incident. Create a change request now: Subject—upgrade network infrastructure; Description—implement redundant network connections to prevent future connectivity issues; Priority—High; Impact—Low; Risk Level—Medium. Link the CR back to the incident and update the incident status to in progress.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_003","query_templated":"After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.","query":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nWe’ve identified what caused the “network connectivity failures across regional offices” incident. Create a change request now: Subject—upgrade network infrastructure; Description—implement redundant network connections to prevent future connectivity issues; Priority—High; Impact—Low; Risk Level—Medium. Link the CR back to the incident and update the incident status to in progress.\n```","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"network connectivity failures across regional offices","change_request_subject":"upgrade network infrastructure","change_request_priority":"High","change_request_impact":"Low","change_request_risk_level":"Medium","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"network connectivity failures across regional offices","change_request_details":"Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/297"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L605","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_change_request_link_incident","data_retrieval_spec":[{"type":"soql","id":"change_request_info","params":{"query":"SELECT Id, ChangeRequestNumber, Impact, Priority, RiskLevel FROM ChangeRequest WHERE Subject='{change_request_subject}'"}},{"type":"soql","id":"incident_id_info","params":{"query":"SELECT RelatedIssueId FROM ChangeRequestRelatedIssue WHERE ChangeRequestId='{retrieved_results['change_request_info'].records[0]['Id']}' AND RelatedEntityType='Incident'"}},{"type":"soql","id":"incident_info","params":{"query":"SELECT Id, Subject, Status FROM Incident WHERE Subject='{incident_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"d154b8e70f729ecafa8ea18211655e171c244ad504abfba31c5301d77f2af023"} {"id":"scuba:service_014_004","source_id":"scuba","original_id":"service_014_004","kind":"benchmark_task","instruction":"Root cause nailed down for the “security breach in user authentication system” incident. Please do the following: 1) Create a change request with Subject: enhance security protocols. 2) Description: implement multi-factor authentication and security monitoring tools. 3) Set Priority: Medium, Impact: Medium, Risk Level: Low. 4) Link the change to the incident record and update the incident status to closed.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_004","query_templated":"After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Medium, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.","query":"Root cause nailed down for the “security breach in user authentication system” incident. Please do the following: 1) Create a change request with Subject: enhance security protocols. 2) Description: implement multi-factor authentication and security monitoring tools. 3) Set Priority: Medium, Impact: Medium, Risk Level: Low. 4) Link the change to the incident record and update the incident status to closed.","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"security breach in user authentication system","change_request_subject":"enhance security protocols","change_request_priority":"Moderate","change_request_impact":"Medium","change_request_risk_level":"Low","incident_new_status":"Closed"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"security breach in user authentication system","change_request_details":"Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Medium, Impact: Medium, Risk Level: Low","new_status":"closed"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/298","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident management","source_difficulty":"medium","template_id":"service_014","template_name":"create_change_request_link_incident","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/298"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/298"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_change_request_link_incident"}],"source_schema_notes":[],"template_registry":{"name":"create_change_request_link_incident","original":{"template_id":"service_013","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["incidents.json"]}}},"objects":["ChangeRequest"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nRoot cause nailed down for the “security breach in user authentication system” incident. Please do the following: 1) Create a change request with Subject: enhance security protocols. 2) Description: implement multi-factor authentication and security monitoring tools. 3) Set Priority: Medium, Impact: Medium, Risk Level: Low. 4) Link the change to the incident record and update the incident status to closed.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_004","query_templated":"After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Medium, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.","query":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nRoot cause nailed down for the “security breach in user authentication system” incident. Please do the following: 1) Create a change request with Subject: enhance security protocols. 2) Description: implement multi-factor authentication and security monitoring tools. 3) Set Priority: Medium, Impact: Medium, Risk Level: Low. 4) Link the change to the incident record and update the incident status to closed.\n```","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"security breach in user authentication system","change_request_subject":"enhance security protocols","change_request_priority":"Moderate","change_request_impact":"Medium","change_request_risk_level":"Low","incident_new_status":"Closed"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"security breach in user authentication system","change_request_details":"Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Medium, Impact: Medium, Risk Level: Low","new_status":"closed"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/298"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L605","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_change_request_link_incident","data_retrieval_spec":[{"type":"soql","id":"change_request_info","params":{"query":"SELECT Id, ChangeRequestNumber, Impact, Priority, RiskLevel FROM ChangeRequest WHERE Subject='{change_request_subject}'"}},{"type":"soql","id":"incident_id_info","params":{"query":"SELECT RelatedIssueId FROM ChangeRequestRelatedIssue WHERE ChangeRequestId='{retrieved_results['change_request_info'].records[0]['Id']}' AND RelatedEntityType='Incident'"}},{"type":"soql","id":"incident_info","params":{"query":"SELECT Id, Subject, Status FROM Incident WHERE Subject='{incident_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"3ef64efd5a61aa133d436a4d533c6569344654f5f7dbc84026fd019926114df6"} {"id":"scuba:service_014_005","source_id":"scuba","original_id":"service_014_005","kind":"benchmark_task","instruction":"Good news—we’ve pinned down the cause behind the “database performance degradation in reporting systems” incident. Log a change request with Subject: optimize database performance and Description: implement database indexing and query optimization to improve reporting speed. Set Priority: Low, Impact: High, and Risk Level: High. Tie this change to the incident case and switch the incident status to in progress.","platforms":["Web"],"apps":["Salesforce"],"source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","revision":"b893e2239b035f93d5753a32557346d15935d596","source_file_sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","license":"Apache-2.0","execution_status":"imported_not_run","taxonomy_status":"unmapped","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_005","query_templated":"After an intensive investigation, the root cause for the incident 'database performance degradation in reporting systems' is found. We need to create a change request with the following details. Subject: optimize database performance, Description: implement database indexing and query optimization to improve reporting speed, Priority: Low, Impact: High, Risk Level: High. Link this change request to the incident case and update the incident status to in progress.","query":"Good news—we’ve pinned down the cause behind the “database performance degradation in reporting systems” incident. Log a change request with Subject: optimize database performance and Description: implement database indexing and query optimization to improve reporting speed. Set Priority: Low, Impact: High, and Risk Level: High. Tie this change to the incident case and switch the incident status to in progress.","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"database performance degradation in reporting systems","change_request_subject":"optimize database performance","change_request_priority":"Low","change_request_impact":"High","change_request_risk_level":"High","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"database performance degradation in reporting systems","change_request_details":"Subject: optimize database performance, Description: implement database indexing and query optimization to improve reporting speed, Priority: Low, Impact: High, Risk Level: High","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null},"instruction_language":"en","instruction_origin":"query_field","instruction_selection":"zero_shot_collection_choice_not_cli_default","primary_variant":"zero_shot","source_json_pointer":"/299","source_record_type":"crm_workflow_instance","category":"service","role":"service","subcategory":"Incident management","source_difficulty":"medium","template_id":"service_014","template_name":"create_change_request_link_incident","evaluation_mode":"state_milestones","application_metadata_kind":"source_declared_benchmark_environment","source_material_refs":[{"role":"canonical_task","source_path":"data/test_zero_shot.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_zero_shot.json","sha256":"23c28e05f66aa22c080777583d1c9131238e6e6f476581676836cd95d70154ca","source_json_pointer":"/299"},{"role":"demo_augmented_variant","source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/299"},{"role":"external_template_description","source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident"},{"role":"state_retrieval_specification","source_path":"scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","sha256":"8aeffcdec74adc8870a6cb1c2e191a387d6f97f8a172619f088b5168f6727351","source_json_pointer":"/create_change_request_link_incident"}],"source_schema_notes":[],"template_registry":{"name":"create_change_request_link_incident","original":{"template_id":"service_013","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["incidents.json"]}}},"objects":["ChangeRequest"],"metadata_types":[]},"source_path":"scuba/task_generators/templates_info_service.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/task_generators/templates_info_service.json","sha256":"b188d33e7ff09824efb5caeb840562e08cfe45ec3fb3a44a048e9d9f788f84b6","source_json_pointer":"/create_change_request_link_incident","differs_from_embedded_fields":["objects","prerequisites","template_id"]},"prompt_variants":[{"variant":"demo_augmented","instruction":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGood news—we’ve pinned down the cause behind the “database performance degradation in reporting systems” incident. Log a change request with Subject: optimize database performance and Description: implement database indexing and query optimization to improve reporting speed. Set Priority: Low, Impact: High, and Risk Level: High. Tie this change to the incident case and switch the incident status to in progress.\n```","memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.","memory_content_status":"example_text_present","original":{"category":"service","subcategory":"Incident management","task_id":"service_014_005","query_templated":"After an intensive investigation, the root cause for the incident 'database performance degradation in reporting systems' is found. We need to create a change request with the following details. Subject: optimize database performance, Description: implement database indexing and query optimization to improve reporting speed, Priority: Low, Impact: High, Risk Level: High. Link this change request to the incident case and update the incident status to in progress.","query":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used.\n\n## Your ultimate task\n```\nGood news—we’ve pinned down the cause behind the “database performance degradation in reporting systems” incident. Log a change request with Subject: optimize database performance and Description: implement database indexing and query optimization to improve reporting speed. Set Priority: Low, Impact: High, and Risk Level: High. Tie this change to the incident case and switch the incident status to in progress.\n```","human_review_comment":"tweak","ground_truth_dict":{"incident_subject":"database performance degradation in reporting systems","change_request_subject":"optimize database performance","change_request_priority":"Low","change_request_impact":"High","change_request_risk_level":"High","incident_new_status":"In Progress"},"difficulty":"medium","query_template_name":"create_change_request_link_incident","query_template_metadata":{"template_id":"service_014","template_string":"After an intensive investigation, the root cause for the incident '{incident_subject}' is found. We need to create a change request with the following details. {change_request_details}. Link this change request to the incident case and update the incident status to {new_status}.","prerequisites":{"data":{"objects":{"Incident":["Incident.json"]}}},"objects":["ChangeRequest","ChangeRequestRelatedIssue","Incident"],"metadata_types":[]},"instance_dict":{"incident_subject":"database performance degradation in reporting systems","change_request_details":"Subject: optimize database performance, Description: implement database indexing and query optimization to improve reporting speed, Priority: Low, Impact: High, Risk Level: High","new_status":"in progress"},"on_start":null,"on_finish":null,"in_domain":true,"has_annotation":true,"additional_info":null,"memory":"## Similar task and experience\n\nSample 1:\nTASK:After an intensive investigation, the root cause for the incident 'network connectivity failures across regional offices' is found. We need to create a change request with the following details. Subject: upgrade network infrastructure, Description: implement redundant network connections to prevent future connectivity issues, Priority: High, Impact: Low, Risk Level: Medium. Link this change request to the incident case and update the incident status to in progress.\nEXPERIENCE:\nSuccessful plan\n\n- Log in to the system.\n- Open the App Launcher, search for “Incidents,” and navigate to the Incidents app/record.\n- On the incident record, open the Related tab.\n- In the Change Requests related list, click New Change Request, then select Add New Change Request if prompted. (Creating from the incident’s Related tab automatically links the change request to this incident.)\n- Populate fields:\n - Subject: Upgrade network infrastructure\n - Description: Implement redundant network connections to prevent future connectivity issues\n - Priority: High\n - Impact: Low\n - Risk Level: Medium\n- Save the change request. If a confirmation save is prompted, click Save again.\n- Update the incident status:\n - Click Edit Incident Status\n - Set Status to In Progress\n - Click Save\n\nNotes\n- No hot-keys were used in the successful execution.\nSample 2:\nTASK:After an intensive investigation, the root cause for the incident 'security breach in user authentication system' is found. We need to create a change request with the following details. Subject: enhance security protocols, Description: implement multi-factor authentication and security monitoring tools, Priority: Moderate, Impact: Medium, Risk Level: Low. Link this change request to the incident case and update the incident status to closed.\nEXPERIENCE:\nSuccessful plan:\n\n- Log in to the platform.\n- Open App Launcher, search for “Incidents,” and open the Incidents app/list.\n- Open the target incident and switch to the Related tab.\n- In the Change Requests related list, create a new change request (Add New Change Request), which links it to the incident automatically.\n- Populate fields:\n - Subject: Enhance security protocols\n - Description: Implement multi-factor authentication and security monitoring tools.\n - Priority: Moderate\n - Impact: Medium\n - Risk Level: Low\n- Save the change request.\n- Edit the incident, set Status to Closed, and save.\n\nNotes:\n- No hot-keys were used."},"source_path":"data/test_demo_aug.json","source_url":"https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/data/test_demo_aug.json","source_file_sha256":"e1204732ed4696d0f2bf6c74822d120ea1b79d33f202d0e5407581f6aeb09c7a","source_json_pointer":"/299"}],"environment":{"type":"salesforce_developer_org","interface_policy":"browser_ui","agent_os":"unspecified","requires_external_org":true,"application_label_scope":"benchmark_environment_not_observed_usage","setup_materialized_by_collection":false,"full_environment_mirrored":false},"verifier":{"status":"source_preserved","unresolved_symbols":[],"refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/master_evaluator.py#L92","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/milestone_evaluator_service.py#L605","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/evaluation_data_retrieval_workflows_service.json","https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/scuba/phases/evaluation/retrieve_data_for_evaluation.py#L49"],"binding_scope":"Static name/signature binding; no grader correctness or runtime validation.","scorecard":{"score_round_digits":2,"task_complete_rule":"rounded_score_equals_one"},"reporting_caveat":{"field":"trajectory.is_successful","note":"The BrowserUse wrapper derives this auxiliary field from a local score initialized to zero. Primary ScoreCard/evaluation_result is separate; no runtime incidence measured.","refs":["https://github.com/SalesforceAIResearch/SCUBA/blob/b893e2239b035f93d5753a32557346d15935d596/main_bu.py"]},"kind":"salesforce_state_milestones","method":"MilestoneEvaluator.evaluate_template_create_change_request_link_incident","data_retrieval_spec":[{"type":"soql","id":"change_request_info","params":{"query":"SELECT Id, ChangeRequestNumber, Impact, Priority, RiskLevel FROM ChangeRequest WHERE Subject='{change_request_subject}'"}},{"type":"soql","id":"incident_id_info","params":{"query":"SELECT RelatedIssueId FROM ChangeRequestRelatedIssue WHERE ChangeRequestId='{retrieved_results['change_request_info'].records[0]['Id']}' AND RelatedEntityType='Incident'"}},{"type":"soql","id":"incident_info","params":{"query":"SELECT Id, Subject, Status FROM Incident WHERE Subject='{incident_subject}'"}}],"scoring_scope":"Post-task Salesforce state predicates and milestone weights; not proof of a particular GUI action path."},"instruction_fingerprint":"50f5514d9d30f1022b2433e60e07842326d91bc01e37a23da812353e98593933"}