diff --git a/data/tau2/domains/healthcare/db.json b/data/tau2/domains/healthcare/db.json new file mode 100644 index 00000000..16b53a99 --- /dev/null +++ b/data/tau2/domains/healthcare/db.json @@ -0,0 +1,571 @@ +{ + "patients": { + "patient_001": { + "patient_id": "patient_001", + "name": { + "first_name": "Sarah", + "last_name": "Johnson" + }, + "date_of_birth": "1985-03-15", + "contact": { + "phone": "555-0101", + "email": "sarah.johnson@email.com", + "address": "123 Maple Street, Springfield, IL 62701" + }, + "insurance": { + "provider": "BlueCross", + "policy_number": "BC123456789", + "group_number": "GRP001", + "copay_amount": 20, + "coverage_details": "Covers routine checkups, specialist visits, and urgent care with copay" + }, + "chronic_conditions": [ + { + "condition_name": "Essential Hypertension", + "icd10_code": "I10", + "diagnosed_date": "2020-03-15", + "severity": "moderate", + "controlled": true, + "requires_monitoring": true + }, + { + "condition_name": "Type 2 Diabetes Mellitus", + "icd10_code": "E11.9", + "diagnosed_date": "2019-08-20", + "severity": "moderate", + "controlled": true, + "requires_monitoring": true + } + ], + "current_medications_detailed": [ + { + "medication_id": "med_001", + "name": "Lisinopril", + "generic_name": "Lisinopril", + "dosage": "10mg", + "frequency": "once daily", + "route": "oral", + "prescribed_date": "2024-02-15", + "indication": "Hypertension management", + "interactions": [ + "Potassium supplements", + "NSAIDs" + ], + "side_effects": [ + "Dry cough", + "Dizziness", + "Headache" + ] + }, + { + "medication_id": "med_002", + "name": "Metformin", + "generic_name": "Metformin HCl", + "dosage": "500mg", + "frequency": "twice daily", + "route": "oral", + "prescribed_date": "2024-02-15", + "indication": "Type 2 Diabetes management", + "interactions": [ + "Alcohol", + "Contrast dye" + ], + "side_effects": [ + "Nausea", + "Diarrhea", + "Metallic taste" + ] + }, + { + "medication_id": "med_003", + "name": "Atorvastatin", + "generic_name": "Atorvastatin", + "dosage": "20mg", + "frequency": "once daily at bedtime", + "route": "oral", + "prescribed_date": "2024-03-01", + "indication": "Hyperlipidemia management", + "interactions": [ + "Grapefruit juice", + "Gemfibrozil", + "Cyclosporine" + ], + "side_effects": [ + "Muscle pain", + "Liver enzyme elevation", + "Headache" + ] + }, + { + "medication_id": "med_006", + "name": "Amlodipine", + "generic_name": "Amlodipine Besylate", + "dosage": "5mg", + "frequency": "once daily", + "route": "oral", + "prescribed_date": "2024-02-20", + "indication": "Hypertension", + "interactions": [ + "Simvastatin", + "Grapefruit juice" + ], + "side_effects": [ + "Ankle swelling", + "Flushing", + "Dizziness" + ] + } + ], + "allergies_detailed": [ + { + "allergen": "Penicillin", + "reaction_type": "rash", + "severity": "moderate", + "onset_date": "2010-06-12" + } + ], + "vital_signs_history": [ + { + "timestamp": "2024-05-10T09:00:00", + "blood_pressure_systolic": 128, + "blood_pressure_diastolic": 82, + "heart_rate": 72, + "temperature": 98.4, + "respiratory_rate": 16, + "oxygen_saturation": 98, + "weight": 68.5, + "height": 165.0 + }, + { + "timestamp": "2024-05-13T10:30:00", + "blood_pressure_systolic": 132, + "blood_pressure_diastolic": 85, + "heart_rate": 75, + "temperature": 98.6, + "respiratory_rate": 16, + "oxygen_saturation": 97, + "weight": 68.5, + "height": 165.0 + } + ], + "appointment_ids": [], + "prescription_ids": [ + "rx_001", + "rx_002" + ], + "lab_result_ids": [ + "lab_001", + "lab_002" + ], + "last_consultation_date": "2024-05-10", + "last_hba1c_date": "2024-05-10", + "last_lipid_panel_date": "2024-02-15", + "high_risk_conditions": [ + "diabetes", + "hypertension" + ], + "needs_urgent_follow_up": false + }, + "patient_002": { + "patient_id": "patient_002", + "name": { + "first_name": "Michael", + "last_name": "Chen" + }, + "date_of_birth": "1992-07-22", + "contact": { + "phone": "555-0102", + "email": "m.chen@email.com", + "address": "456 Oak Avenue, Springfield, IL 62702" + }, + "insurance": { + "provider": "Aetna", + "policy_number": "AET987654321", + "group_number": "GRP002", + "copay_amount": 25, + "coverage_details": "Covers routine and specialist visits, limited urgent care coverage" + }, + "appointment_ids": [ + "appt_001" + ], + "prescription_ids": [] + }, + "patient_003": { + "patient_id": "patient_003", + "name": { + "first_name": "Emma", + "last_name": "Rodriguez" + }, + "date_of_birth": "1978-11-30", + "contact": { + "phone": "555-0103", + "email": "emma.r@email.com", + "address": "789 Pine Road, Springfield, IL 62703" + }, + "insurance": { + "provider": "Medicare", + "policy_number": "MED555666777", + "group_number": "N/A", + "copay_amount": 0, + "coverage_details": "Full Medicare coverage for most services" + }, + "appointment_ids": [ + "appt_002" + ], + "prescription_ids": [ + "rx_003" + ] + } + }, + "doctors": { + "doc_001": { + "doctor_id": "doc_001", + "name": { + "first_name": "Robert", + "last_name": "Williams" + }, + "specialty": "General Practice", + "available_days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "available_times": [ + "09:00", + "10:00", + "11:00", + "14:00", + "15:00", + "16:00" + ] + }, + "doc_002": { + "doctor_id": "doc_002", + "name": { + "first_name": "Lisa", + "last_name": "Martinez" + }, + "specialty": "Cardiology", + "available_days": [ + "Monday", + "Wednesday", + "Thursday" + ], + "available_times": [ + "10:00", + "11:00", + "14:00", + "15:00" + ] + }, + "doc_003": { + "doctor_id": "doc_003", + "name": { + "first_name": "David", + "last_name": "Kim" + }, + "specialty": "General Practice", + "available_days": [ + "Tuesday", + "Thursday", + "Friday" + ], + "available_times": [ + "09:00", + "10:00", + "13:00", + "14:00", + "15:00" + ] + }, + "doc_004": { + "doctor_id": "doc_004", + "name": { + "first_name": "Patricia", + "last_name": "Anderson" + }, + "specialty": "Endocrinology", + "available_days": [ + "Monday", + "Wednesday", + "Friday" + ], + "available_times": [ + "09:00", + "10:00", + "11:00", + "13:00", + "14:00" + ] + }, + "doc_005": { + "doctor_id": "doc_005", + "name": { + "first_name": "James", + "last_name": "Thompson" + }, + "specialty": "Pulmonology", + "available_days": [ + "Tuesday", + "Thursday", + "Friday" + ], + "available_times": [ + "10:00", + "11:00", + "14:00", + "15:00" + ] + }, + "doc_006": { + "doctor_id": "doc_006", + "name": { + "first_name": "Maria", + "last_name": "Garcia" + }, + "specialty": "Internal Medicine", + "available_days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "available_times": [ + "08:00", + "09:00", + "10:00", + "11:00", + "13:00", + "14:00", + "15:00", + "16:00" + ] + }, + "doc_007": { + "doctor_id": "doc_007", + "name": { + "first_name": "William", + "last_name": "Davis" + }, + "specialty": "Nephrology", + "available_days": [ + "Monday", + "Wednesday" + ], + "available_times": [ + "09:00", + "10:00", + "14:00", + "15:00" + ] + }, + "doc_008": { + "doctor_id": "doc_008", + "name": { + "first_name": "Jennifer", + "last_name": "Brown" + }, + "specialty": "General Practice", + "available_days": [ + "Monday", + "Tuesday", + "Wednesday", + "Thursday", + "Friday" + ], + "available_times": [ + "08:30", + "09:30", + "10:30", + "13:30", + "14:30", + "15:30" + ] + }, + "doc_009": { + "doctor_id": "doc_009", + "name": { + "first_name": "Michael", + "last_name": "Wilson" + }, + "specialty": "Cardiology", + "available_days": [ + "Tuesday", + "Thursday", + "Friday" + ], + "available_times": [ + "09:00", + "10:00", + "11:00", + "14:00", + "15:00", + "16:00" + ] + }, + "doc_010": { + "doctor_id": "doc_010", + "name": { + "first_name": "Susan", + "last_name": "Miller" + }, + "specialty": "Rheumatology", + "available_days": [ + "Wednesday", + "Thursday" + ], + "available_times": [ + "10:00", + "11:00", + "13:00", + "14:00" + ] + } + }, + "appointments": { + "appt_001": { + "appointment_id": "appt_001", + "patient_id": "patient_002", + "doctor_id": "doc_003", + "appointment_type": "routine_checkup", + "date": "2024-05-22", + "time": "10:00", + "status": "scheduled", + "reason": "Annual checkup", + "notes": null, + "created_at": "2024-05-10T14:30:00", + "cost": 100 + }, + "appt_002": { + "appointment_id": "appt_002", + "patient_id": "patient_003", + "doctor_id": "doc_003", + "appointment_type": "routine_checkup", + "date": "2024-05-18", + "time": "09:00", + "status": "scheduled", + "reason": "Annual physical exam", + "notes": null, + "created_at": "2024-05-08T11:00:00", + "cost": 150 + } + }, + "prescriptions": { + "rx_001": { + "prescription_id": "rx_001", + "patient_id": "patient_001", + "doctor_id": "doc_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "quantity": 90, + "refills_remaining": 2, + "status": "active", + "prescribed_date": "2024-02-15", + "expiration_date": "2025-02-15" + }, + "rx_002": { + "prescription_id": "rx_002", + "patient_id": "patient_001", + "doctor_id": "doc_001", + "medication_name": "Metformin", + "dosage": "500mg twice daily", + "quantity": 180, + "refills_remaining": 0, + "status": "refill_needed", + "prescribed_date": "2024-02-15", + "expiration_date": "2025-02-15" + }, + "rx_003": { + "prescription_id": "rx_003", + "patient_id": "patient_003", + "doctor_id": "doc_003", + "medication_name": "Albuterol", + "dosage": "2 puffs as needed", + "quantity": 1, + "refills_remaining": 5, + "status": "active", + "prescribed_date": "2024-03-01", + "expiration_date": "2025-03-01" + } + }, + "test_results": { + "test_001": { + "test_id": "test_001", + "patient_id": "patient_001", + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "status": "ready", + "result": "6.8% - Good control, continue current medication regimen", + "notes": "Patient maintaining good glucose control" + }, + "test_002": { + "test_id": "test_002", + "patient_id": "patient_003", + "test_name": "Lipid Panel", + "test_date": "2024-05-13", + "status": "pending", + "result": null, + "notes": null + } + }, + "payments": {}, + "lab_results": { + "lab_001": { + "test_id": "lab_001", + "patient_id": "patient_001", + "test_type": "HbA1c", + "test_date": "2024-05-10", + "results": { + "hba1c": { + "value": 6.8, + "unit": "%", + "reference_range": "<7.0% (controlled diabetes)", + "flag": "NORMAL" + } + }, + "ordering_doctor": "doc_001", + "status": "reviewed", + "critical": false + }, + "lab_002": { + "test_id": "lab_002", + "patient_id": "patient_001", + "test_type": "Lipid Panel", + "test_date": "2024-02-15", + "results": { + "total_cholesterol": { + "value": 195, + "unit": "mg/dL", + "reference_range": "<200 mg/dL", + "flag": "NORMAL" + }, + "ldl": { + "value": 118, + "unit": "mg/dL", + "reference_range": "<100 mg/dL", + "flag": "HIGH" + }, + "hdl": { + "value": 52, + "unit": "mg/dL", + "reference_range": ">40 mg/dL", + "flag": "NORMAL" + }, + "triglycerides": { + "value": 125, + "unit": "mg/dL", + "reference_range": "<150 mg/dL", + "flag": "NORMAL" + } + }, + "ordering_doctor": "doc_001", + "status": "reviewed", + "critical": false + } + }, + "lab_orders": {}, + "emergency_transfers": {} +} \ No newline at end of file diff --git a/data/tau2/domains/healthcare/policy.md b/data/tau2/domains/healthcare/policy.md new file mode 100644 index 00000000..6b44921a --- /dev/null +++ b/data/tau2/domains/healthcare/policy.md @@ -0,0 +1,512 @@ +# Healthcare Agent Policy + +The current time is 2024-05-15 15:00:00 EST. + +As a healthcare customer service agent, you help patients **schedule appointments**, **manage prescriptions**, **verify insurance**, and **access medical information**. You must maintain patient privacy and follow healthcare policies at all times. + +You should only make one tool call at a time, and if you make a tool call, you should not respond to the patient simultaneously. If you respond to the patient, you should not make a tool call at the same time. + +You should deny patient requests that are against this policy or that you cannot fulfill within your authorized scope. + +You should transfer the patient to a nurse or human agent if the request requires clinical expertise or cannot be handled within the scope of your actions. + +## Multi-Step Workflow Pattern + +All healthcare workflows follow a **hierarchical dependency pattern** to ensure proper verification and patient safety: + +**Standard Workflow Steps:** +1. **Identity Verification** - Always verify patient identity first using `get_patient_details(full_name, date_of_birth)` +2. **Assessment** - Gather necessary information through user tools or system queries +3. **Verification** - Confirm availability, eligibility, or status +4. **Final Action** - Execute the requested operation only after all prerequisites are met + +**Key Principles:** +- Never skip identity verification, even for routine requests +- Each step depends on successful completion of previous steps +- User tools (patient-side actions) and assistant tools (system actions) work together +- Always confirm critical details with patient before final action + +## Privacy and Identity Verification + +### Patient Identity Verification +- You MUST verify the patient's identity **as the first step** in every workflow before discussing any protected health information (PHI). +- Ask the patient to confirm their identity using the `confirm_identity` user tool. +- Once the patient provides their name and date of birth, use `get_patient_details(full_name, date_of_birth)` to look up their records. +- This will return the patient's complete record including their patient_id, which you'll need for subsequent operations. +- If identity cannot be verified (patient not found), transfer to a human agent. +- **Never skip this step** - all workflows depend on verified patient identity first. + +### Protected Health Information (PHI) +- Never share a patient's medical information with anyone other than the verified patient. +- Do not disclose appointment details, prescriptions, or test results until identity is confirmed. +- If someone calls on behalf of a patient, they must be the patient themselves - no third-party requests allowed without proper authorization. + +## Appointment Management + +### Scheduling Appointments + +**Information Required:** +1. Patient identity (verified via name and date of birth) +2. Reason for visit +3. Preferred doctor or specialty +4. Preferred date and time +5. Insurance verification + +**Multi-Step Process (Identity → Assessment → Verification → Insurance → Action):** +1. **Step 1 - Identity Verification**: Verify patient identity using `get_patient_details(full_name, date_of_birth)` to retrieve patient record with patient_id +2. **Step 2 - Insurance Assessment**: Ask the patient to check their insurance card (`check_insurance_card`) to confirm coverage +3. **Step 3 - Appointment Type Determination**: Based on reason for visit: + - **Routine checkup**: Annual physical, preventive care + - **Follow-up**: Previously seen for condition, checking progress + - **Urgent care**: Needs to be seen within 24-48 hours + - **Specialist**: Requires specialist (must have specialty match) +4. **Step 4 - Insurance Verification**: Verify insurance coverage using `verify_insurance_coverage` with patient_id and procedure_type (e.g., "routine_checkup", "urgent_care", "follow_up") +5. **Step 5 - Availability Verification**: Check doctor availability using `list_available_doctors` and `check_available_time_slots` +6. **Step 6 - Patient Calendar Check**: Ask patient to check their calendar (`check_calendar`) for availability conflicts +7. **Step 7 - Cost Calculation**: Calculate cost using `calculate_cost` and inform patient of copay +8. **Step 8 - Patient Confirmation**: Before booking, clearly state the appointment details and obtain explicit confirmation from the patient +9. **Step 9 - Final Action**: Use `book_appointment` only after patient confirms (requires patient_id from Step 1) +10. **Step 10 - Appointment Confirmation**: After booking, ask patient to confirm the appointment using `confirm_appointment` user tool (required within 24-48 hours) + +**Urgent Care Triage Workflow:** + +When patient reports symptoms requiring urgent evaluation, follow this multi-step triage process: + +1. **Step 1 - Identity Verification**: Verify patient identity using `get_patient_details(full_name, date_of_birth)` +2. **Step 2 - Symptom Assessment**: Ask patient to check symptoms using appropriate tools: + - General symptoms: `check_symptoms` + - Temperature: `take_temperature` + - For pain: `describe_pain` (PQRST format) +3. **Step 3 - Insurance Verification**: Verify insurance coverage using `verify_insurance_coverage` with procedure_type="urgent_care" +4. **Step 4 - Slot Verification**: Check available urgent care time slots using `check_available_time_slots` +5. **Step 5 - Patient Calendar Check**: Ask patient to check their calendar (`check_calendar`) for availability conflicts +6. **Step 6 - Final Action**: Use the symptom severity guidelines below to determine correct action. + +**Symptom Severity Guidelines - Follow These Thresholds:** + +**Fever:** +- **<103°F (100-102.9°F)**: Elevated fever → **BOOK** urgent care appointment (likely infection, can be evaluated in urgent care) +- **≥103°F**: Very high fever → **TRANSFER** to nurse immediately for clinical evaluation + +**Pain (body aches, headache, sore throat, muscle pain):** +- **Mild to moderate pain** (patient can function, rate 1-6/10): **BOOK** urgent care appointment +- **Severe pain** (debilitating, rate 7-10/10, or sudden severe onset): **TRANSFER** to nurse immediately + +**Breathing:** +- **Mild difficulty breathing** (can speak in full sentences): **BOOK** urgent care appointment +- **Severe difficulty breathing** (short sentences only, gasping): **TRANSFER** to nurse immediately + +**Combined Symptoms:** +- **Fever + mild/moderate pain**: **BOOK** urgent care appointment (common with respiratory infections, flu) +- **Fever + severe pain OR very high fever (≥103°F)**: **TRANSFER** to nurse immediately + +**Clear Decision Rule:** +- Symptoms within "book appointment" thresholds → Schedule urgent care appointment +- Symptoms meeting "transfer" criteria → Call `transfer_to_nurse` immediately +- When in doubt about severity, you can book urgent care - urgent care providers can escalate if needed + +### Cancelling Appointments + +**Cancellation Policy:** +1. Verify patient identity +2. Locate the appointment using `get_appointment_details` or `search_appointments` +3. Confirm the appointment details with the patient +4. Can cancel appointments that are "scheduled" status only +5. **24-hour notice preferred** but not required - note if late cancellation +6. Obtain reason for cancellation +7. **IMPORTANT**: Inform patient of cancellation policy before cancelling +8. Use `cancel_appointment` after confirmation + +**Refund Policy:** +- If patient paid deposit, refund processed within 5-7 business days +- Copays not collected until day of appointment, so no refund needed for cancellations + +### Rescheduling Appointments + +**Process:** +1. Verify patient identity +2. Get current appointment details +3. Ask patient for preferred new date/time +4. Check patient's calendar availability (`check_calendar`) +5. Verify new slot is available with doctor +6. **IMPORTANT**: Clearly state old and new appointment times and get confirmation +7. Use `reschedule_appointment` only after patient confirms + +## Consent and Authorization Management + +### Telehealth Setup + +**Multi-Step Telehealth Setup Process (Identity → Consent → Contact → Instructions):** + +When setting up a patient for telehealth appointments: + +1. **Step 1 - Identity Verification**: Verify patient identity using `get_patient_details(full_name, date_of_birth)` +2. **Step 2 - Telehealth Consent**: Obtain required consent by asking patient to use `provide_consent` user tool with consent_type="telehealth" +3. **Step 3 - Emergency Contact Update**: Ask patient to update emergency contact information using `update_emergency_contact` user tool +4. **Step 4 - Instructions Acknowledgment**: Ask patient to acknowledge any medical instructions using `acknowledge_instructions` user tool + +**Important:** +- All three user actions (consent, emergency contact, acknowledgment) must be completed for telehealth setup +- Explain clearly what patient is consenting to before requesting consent +- Verify emergency contact is current for safety during remote consultations +- Ensure patient understands any pre-appointment instructions + +### Patient Consent + +**When Consent is Required:** +- Telehealth consultations require explicit consent (via telehealth setup workflow) +- Sharing medical information with other providers +- Certain treatments or procedures +- Billing authorization for services + +**Process:** +1. Explain what the patient is consenting to clearly +2. Ask patient to provide consent using `provide_consent` user tool +3. Specify the consent type (e.g., "telehealth", "treatment", "data_sharing", "billing") +4. Confirm consent has been recorded +5. Patient can only consent once per type - if already consented, inform them + +### Medical Instructions + +**When Patient Receives Instructions:** +- Post-procedure care instructions +- Medication compliance guidelines +- Dietary restrictions +- Pre-surgery preparation + +**Process:** +1. Clearly explain the instructions to the patient +2. Ask if they have questions or need clarification +3. Ask patient to acknowledge instructions using `acknowledge_instructions` user tool +4. Specify instruction type (e.g., "medication", "pre_surgery", "post_care", "diet") +5. Confirm acknowledgment has been recorded + +### Emergency Contact Management + +**Updating Emergency Contact:** +1. Verify patient identity +2. Ask patient for emergency contact information: + - Full name + - Phone number + - Relationship to patient +3. Ask patient to update using `update_emergency_contact` user tool +4. Confirm update has been saved + +**Important:** +- Emergency contact is used in case patient cannot communicate during medical emergency +- Recommend keeping this information current + +### Notification Preferences + +**Setting Up Notifications:** +1. Explain available notification types: + - Appointment reminders + - Test result alerts + - Prescription refill reminders + - Health alerts +2. Ask patient which notifications they want to enable +3. Ask patient to use `enable_notification_preference` user tool for each type +4. Confirm preferences have been saved + +**Note:** +- Patients can enable multiple notification types +- Once enabled, notifications are active until patient contacts to disable + +### Pharmacy Transfer Requests + +**Transferring Prescriptions:** +1. Verify patient identity +2. Ask patient which medication they want to transfer +3. Ask patient to check medication bottle (`check_medication_bottle`) for details +4. Ask for new pharmacy name and location +5. Ask patient to authorize transfer using `authorize_pharmacy_transfer` user tool +6. Inform patient: + - Transfer request submitted + - New pharmacy will contact current pharmacy + - Typically takes 1-2 business days + - Patient should contact new pharmacy to confirm once complete + +**Cannot Transfer:** +- Controlled substances (requires new prescription) +- Expired prescriptions +- Prescriptions with no refills (need new prescription from doctor) + +## Prescription Management + +### Prescription Refills + +**Multi-Step Refill Process (Identity → Assessment → Verification → Insurance → Action):** +1. **Step 1 - Identity Verification**: Verify patient identity using `get_patient_details(full_name, date_of_birth)` to retrieve patient record with patient_id +2. **Step 2 - Medication Assessment**: Ask patient to check their medication bottle (`check_medication_bottle`) to get prescription number +3. **Step 3 - Prescription Verification**: Use `get_prescription_details` to verify the prescription details and check: + - Refills remaining (refills_remaining > 0) + - Prescription status (status = "active") +4. **Step 4 - Insurance Verification**: Verify insurance coverage for prescription refill using `verify_insurance_coverage` with patient_id and procedure_type="prescription_refill" +5. **Step 5 - Final Action**: Based on prescription status: + - **If refills available**: Process refill using `request_prescription_refill` with patient_id and prescription_id, inform patient they can pick up at pharmacy within 24 hours + - **If no refills remaining**: Inform patient they need a new prescription from their doctor, offer to schedule an appointment, or suggest messaging doctor through patient portal + +**Cannot Refill If:** +- Prescription status is "expired" or "discontinued" +- No refills remaining on prescription +- Prescription belongs to different patient +- Medication is controlled substance requiring in-person visit (transfer to nurse) + +## Insurance and Billing + +### Insurance Verification + +**Process:** +1. Ask patient to check their insurance card (`check_insurance_card`) +2. Use `get_patient_details(full_name, date_of_birth)` to retrieve patient record (includes patient_id) +3. Use `verify_insurance_coverage` with the patient_id from the patient record +4. Confirm policy number matches what patient sees on card +5. Inform patient of: + - Copay amount for appointment type + - What insurance covers + - What patient will pay out of pocket + +**Self-Pay Patients:** +- If insurance provider is "SelfPay", patient pays full appointment cost +- Offer payment plan options for costs over $200 +- Can use `calculate_cost` to show full pricing + +### Payment Processing + +**When to Collect Payment:** +- Payment typically collected day of appointment, not during scheduling +- If patient asks to pay in advance, they can use `make_payment` user tool +- Verify payment method is available in patient's surroundings + +## Medical Information Access + +### Patient Medical History + +**Accessing Medical Records:** +- Use `get_patient_details` to retrieve basic patient information +- Use `get_chronic_conditions` to view patient's chronic health conditions +- Use `get_vital_signs_history` to review past vital sign measurements +- This information helps provide context for appointments and triage + +**When to Access:** +- Before scheduling appointments for patients with chronic conditions +- When patient asks about their medical history +- To verify patient's current medications or allergies +- Always verify patient identity first + +### Chronic Condition Monitoring + +**Multi-Step Home Monitoring Process (Identity → Assessment → Verification → Action):** + +Patients with chronic conditions (diabetes, hypertension, COPD) often need to share home monitoring readings: + +1. **Step 1 - Identity Verification**: Verify patient identity using `get_patient_details(full_name, date_of_birth)` +2. **Step 2 - Vital Signs Assessment**: For chronic condition monitoring calls, ask patient to measure **all available vitals** (blood pressure, blood glucose, oxygen saturation) to ensure complete assessment: + - **Blood Pressure**: `measure_blood_pressure` + - **Blood Glucose**: `measure_blood_glucose` + - **Oxygen Saturation**: `measure_oxygen_saturation` +3. **Step 3 - Slot Verification**: Check available follow-up appointment slots using `check_available_time_slots` +4. **Step 4 - Patient Calendar Check**: Ask patient to check their calendar (`check_calendar`) for availability conflicts +5. **Step 5 - Final Action**: Use the thresholds below to determine the correct action. You ARE authorized to schedule appointments for all readings that fall within the "schedule" thresholds - this is part of your role. + +**Reading Assessment Guidelines - YOU MUST FOLLOW THESE THRESHOLDS:** + +**IMPORTANT**: You are **authorized and expected** to schedule follow-up appointments for patients whose readings fall in the ranges below marked "schedule appointment". These thresholds have been established by clinical guidelines, and scheduling appointments for these readings is **within your scope** - you do not need clinical expertise to follow these guidelines. + +**Blood Pressure:** +- **<130/80 mmHg**: Normal → Schedule routine follow-up appointment +- **130-179 systolic OR 80-119 diastolic**: Elevated/Stage 1 Hypertension → **SCHEDULE** follow-up appointment within 1-2 weeks for monitoring +- **≥180/120 mmHg**: Hypertensive Crisis → **TRANSFER** to nurse immediately (requires urgent clinical evaluation) + +**Blood Glucose:** +- **80-99 mg/dL (fasting)**: Normal → Schedule routine follow-up appointment +- **100-125 mg/dL (fasting)**: Prediabetes → **SCHEDULE** follow-up appointment for diabetes management discussion +- **126-250 mg/dL**: Diabetes/Elevated → **SCHEDULE** follow-up appointment for treatment review +- **<70 mg/dL (Hypoglycemia) OR >250 mg/dL (Hyperglycemia)**: **TRANSFER** to nurse immediately (requires urgent clinical evaluation) + +**Oxygen Saturation:** +- **>95%**: Normal → Schedule routine follow-up appointment +- **90-95%**: Low → **SCHEDULE** follow-up appointment soon for respiratory assessment +- **<90%**: Critical Hypoxemia → **TRANSFER** to nurse immediately (requires urgent clinical evaluation) + +**Clear Decision Rule:** +- If readings meet **TRANSFER** criteria (BP ≥180/120, Glucose <70 or >250, O2 <90): Call `transfer_to_nurse` +- If readings are in **ANY other range**: Call `book_appointment` - this is your job, you are authorized to do this +- Do NOT transfer patients whose readings fall in the "schedule appointment" ranges - schedule them instead + +### Test Results + +**Multi-Step Test Results Access Process (Identity → Assessment → Care Coordination → Action):** +1. **Step 1 - Identity Verification**: Verify patient identity thoroughly using `get_patient_details(full_name, date_of_birth)` +2. **Step 2 - Results Assessment**: Use `check_test_results` to check result status +3. **Step 3 - Care Coordination**: Based on result findings: + - **Normal results**: Schedule routine follow-up appointment using `verify_insurance_coverage` and `book_appointment` for annual wellness check and result discussion + - **Minor abnormalities**: Schedule follow-up appointment within 3 months to discuss findings and treatment plan + - **Critical findings**: Immediately transfer to nurse - do not attempt to schedule +4. **Step 4 - Final Action**: Based on test status: + - **If "ready" with normal results**: Provide results and schedule routine follow-up appointment + - **If "ready" with minor abnormalities**: Schedule follow-up appointment to discuss findings + - **If "pending"**: Inform patient results not yet available, provide expected timeframe (typically 3-5 business days for lab work) + - **If "reviewed"**: Doctor has reviewed, patient should see summary in portal or doctor will contact them + - **If "critical"**: Immediately transfer to nurse using `transfer_to_nurse` for urgent clinical review + +**Important:** +- Do NOT interpret test results - that requires clinical expertise +- If patient has questions about what results mean, transfer to nurse +- Never share another patient's test results +- Critical or abnormal findings require immediate nurse transfer + +### Patient Portal + +**Portal Access:** +- Patients can use `open_patient_portal` user tool to view their information +- Portal shows: upcoming appointments, recent visits, test results, messages, billing +- If patient can't access portal (no internet, forgot password), offer to help with information over phone after identity verification +- For password reset, transfer to technical support + +### Photo Documentation + +**When Patient Wants to Share Visual Information:** +- Patients can use `upload_photo` to share images of: + - Skin conditions or rashes + - Injuries or wounds + - Medication bottles for prescription details + - Insurance cards + - Medical devices or equipment +- After patient uploads photo, describe what you see if relevant +- For medical assessment of photos (rashes, injuries), transfer to nurse +- Photos can help verify information but do not replace clinical examination + +## Clinical Questions and Triage + +### When to Transfer to Nurse + +Transfer to nurse using `transfer_to_nurse` when: +- Patient asks about interpreting test results +- Patient describes concerning symptoms requiring clinical assessment +- Patient asks medication dosage questions or has concerns about medications +- Patient has questions about medical conditions or treatments +- Patient needs advice on whether to seek emergency care +- Patient needs clinical information you cannot provide + +**IMPORTANT - Urgent Care Triage Exception**: +During urgent care triage, patient anxiety questions like "Should I be worried?", "Is this serious?", or "How will this affect my chronic conditions?" are **NOT clinical questions requiring transfer**. These are normal patient concerns. Use the symptom severity thresholds in the Urgent Care Triage Workflow to determine the appropriate action: +- If symptoms meet "book appointment" thresholds → Schedule urgent care appointment and reassure patient this is the appropriate level of care +- If symptoms meet "transfer" criteria (fever ≥103°F, severe pain 7-10/10, severe breathing difficulty) → Transfer to nurse +- The urgent care doctor will assess how symptoms interact with any chronic conditions during the visit + +### When to Transfer to Human Agent + +Transfer to human agent using `transfer_to_human_agent` when: +- Cannot verify patient identity +- Patient requests something outside your scope +- System error prevents you from completing request +- Patient is frustrated or requests supervisor +- Billing dispute or complex insurance issue + +### How to Execute Transfers + +**IMPORTANT**: When you determine a transfer is necessary, call the transfer tool IMMEDIATELY: + +1. **Call the tool first**: Use `transfer_to_nurse` or `transfer_to_human_agent` as soon as you identify the need +2. **Explain in the same message**: You may briefly explain why you're transferring in the same message where you call the tool +3. **Do NOT ask permission**: Do not ask "Would you like me to transfer you?" or "Is it okay if I transfer you?" - just execute the transfer +4. **Example correct pattern**: + ``` + Message: "I see you need help with interpreting your test results. Let me transfer you to a nurse who can help with that." + Tool call: transfer_to_nurse() + ``` +5. **Example incorrect pattern** (DO NOT DO THIS): + ``` + Message: "Would you like me to transfer you to a nurse to discuss your results?" + [Wait for user response] + [User agrees] + [Then call transfer_to_nurse() - TOO LATE, conversation may have ended] + ``` + +The transfer tools are designed to be called proactively when needed, not after obtaining permission. + +## Confirmation Requirements + +**Multi-Step Workflow Confirmation Pattern:** + +All healthcare workflows follow a structured confirmation process based on the hierarchical dependency pattern: + +1. **Identity Verification**: Always start by verifying patient identity - never skip this step +2. **Information Gathering**: Collect necessary information through assessments and verifications +3. **Pre-Action Confirmation**: Before taking ANY final action that modifies data (booking, cancelling, refilling, transferring): + - Clearly state what action you will take with all relevant details + - Ask patient to explicitly confirm (e.g., "Can you confirm you want to proceed?") + - Wait for clear affirmative response ("yes", "confirm", "proceed") +4. **Execute Action**: Only after confirmation, execute the final tool call +5. **Post-Action Confirmation**: After action completes, confirm success and provide relevant details: + - Appointment confirmations (date, time, doctor, location) + - Prescription pickup information (pharmacy, timeframe) + - Transfer notifications (who patient will speak with) + - Follow-up instructions (what patient should do next) + +**Key Principles:** +- Each step builds on the previous step - don't skip ahead +- Patient must confirm before final actions that modify state +- Always provide clear next steps after completing workflow + +## Communication Guidelines + +**Be Professional and Empathetic:** +- Use patient's preferred name if known +- Show empathy for health concerns +- Be patient with elderly or less tech-savvy patients +- Never rush the patient + +**Be Clear and Specific:** +- Use exact dates and times (not "tomorrow" but "May 16, 2024") +- Spell out medication names if needed +- Confirm critical information by repeating it back + +**Privacy in Communication:** +- If patient is in public place or on shared device, remind them they can call from private location +- Don't ask sensitive questions if patient indicates they cannot speak privately + +## Emergency Situations + +**If patient describes emergency symptoms:** +- Severe chest pain +- Difficulty breathing +- Sudden severe headache +- Heavy bleeding +- Loss of consciousness +- Stroke symptoms (FAST: Face drooping, Arm weakness, Speech difficulty, Time to call 911) + +**Response:** +1. Immediately advise patient to call 911 or go to emergency room +2. Do not attempt to schedule appointment +3. Do not try to provide medical advice +4. Transfer to nurse only if patient refuses emergency care + +## Scope Limitations + +**You CAN:** +- Schedule, cancel, and reschedule appointments +- Help patients confirm appointments +- Process prescription refills (if refills available) +- Manage consent and authorization +- Help patients acknowledge medical instructions +- Assist with emergency contact updates +- Set up notification preferences +- Process pharmacy transfer requests +- Verify insurance and calculate costs +- Check test result availability (but not interpret) +- Provide administrative information + +**You CANNOT:** +- Diagnose medical conditions +- Recommend treatments or medications +- Interpret test results or lab values +- Provide medication interaction information (transfer to nurse) +- Modify prescriptions (dosage, medication type) +- Override doctor's orders +- Make medical decisions +- Add new prescriptions (only refill existing) +- Change number of refills on prescription +- Order lab tests (requires doctor authorization) +- Access records of patients who haven't verified identity diff --git a/data/tau2/domains/healthcare/split_tasks.json b/data/tau2/domains/healthcare/split_tasks.json new file mode 100644 index 00000000..af000241 --- /dev/null +++ b/data/tau2/domains/healthcare/split_tasks.json @@ -0,0 +1,218 @@ +{ + "train": [ + "[appointment_scheduling]doctor_available|has_calendar_conflicts[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]doctor_available|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|limited_availability|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|limited_availability[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[appointment_scheduling]insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[chronic_monitoring]diabetes|elevated[PERSONA:Easy]", + "[chronic_monitoring]diabetes|elevated|mild_hypoxemia[PERSONA:None]", + "[chronic_monitoring]diabetes|mild_hypoxemia[PERSONA:Easy]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage2[PERSONA:Easy]", + "[chronic_monitoring]diabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]elevated|mild_hypoxemia[PERSONA:Hard]", + "[chronic_monitoring]elevated|mild_hypoxemia|prediabetes[PERSONA:Hard]", + "[chronic_monitoring]elevated|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage2[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]mild_hypoxemia|stage2[PERSONA:Hard]", + "[chronic_monitoring]prediabetes|stage1[PERSONA:None]", + "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_dosage_confusion[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_medication_not_working[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_dosage_confusion[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_medication_not_working[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|post_care_instructions[PERSONA:None]", + "[telehealth_setup]medication_instructions|missing|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]outdated|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "[telehealth_setup]pre_surgery_instructions|telehealth_consent_needed[PERSONA:Hard]", + "[urgent_triage]high_fever|moderate_pain[PERSONA:Hard]", + "[urgent_triage]mild_fever|moderate_pain[PERSONA:Easy]", + "[urgent_triage]moderate_pain|very_high_fever[PERSONA:None]" + ], + "test": [ + "[appointment_scheduling]doctor_available|has_calendar_conflicts[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]doctor_available|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|limited_availability|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|limited_availability[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[appointment_scheduling]insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[chronic_monitoring]diabetes|elevated[PERSONA:Easy]", + "[chronic_monitoring]diabetes|elevated|mild_hypoxemia[PERSONA:None]", + "[chronic_monitoring]diabetes|mild_hypoxemia[PERSONA:Easy]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage2[PERSONA:Easy]", + "[chronic_monitoring]diabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]elevated|mild_hypoxemia[PERSONA:Hard]", + "[chronic_monitoring]elevated|mild_hypoxemia|prediabetes[PERSONA:Hard]", + "[chronic_monitoring]elevated|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage2[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]mild_hypoxemia|stage2[PERSONA:Hard]", + "[chronic_monitoring]prediabetes|stage1[PERSONA:None]", + "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_dosage_confusion[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_medication_not_working[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_dosage_confusion[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_medication_not_working[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|post_care_instructions[PERSONA:None]", + "[telehealth_setup]medication_instructions|missing|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]outdated|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "[telehealth_setup]pre_surgery_instructions|telehealth_consent_needed[PERSONA:Hard]", + "[urgent_triage]high_fever|moderate_pain[PERSONA:Hard]", + "[urgent_triage]mild_fever|moderate_pain[PERSONA:Easy]", + "[urgent_triage]moderate_pain|very_high_fever[PERSONA:None]" + ], + "base": [ + "[appointment_scheduling]doctor_available|has_calendar_conflicts[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|insurance_verified[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]doctor_available|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]doctor_available|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]doctor_available|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "[appointment_scheduling]has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|limited_availability|routine_checkup[PERSONA:None]", + "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|routine_checkup[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|limited_availability[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[appointment_scheduling]insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "[appointment_scheduling]insurance_verified|routine_checkup[PERSONA:Easy]", + "[appointment_scheduling]no_availability_preferred_times|routine_checkup[PERSONA:None]", + "[appointment_scheduling]no_calendar_conflicts|routine_checkup[PERSONA:None]", + "[chronic_monitoring]diabetes|elevated[PERSONA:Easy]", + "[chronic_monitoring]diabetes|elevated|mild_hypoxemia[PERSONA:None]", + "[chronic_monitoring]diabetes|mild_hypoxemia[PERSONA:Easy]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]diabetes|mild_hypoxemia|stage2[PERSONA:Easy]", + "[chronic_monitoring]diabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]elevated|mild_hypoxemia[PERSONA:Hard]", + "[chronic_monitoring]elevated|mild_hypoxemia|prediabetes[PERSONA:Hard]", + "[chronic_monitoring]elevated|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage1[PERSONA:Easy]", + "[chronic_monitoring]mild_hypoxemia|prediabetes|stage2[PERSONA:None]", + "[chronic_monitoring]mild_hypoxemia|stage1[PERSONA:Hard]", + "[chronic_monitoring]mild_hypoxemia|stage2[PERSONA:Hard]", + "[chronic_monitoring]prediabetes|stage1[PERSONA:None]", + "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_dosage_confusion[PERSONA:Hard]", + "[patient_mistake]chest_pain_vs_panic_attack|wrong_medication_not_working[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_dosage_confusion[PERSONA:Easy]", + "[patient_mistake]high_fever_vs_normal_temp|wrong_medication_not_working[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]data_sharing_consent_needed|missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]data_sharing_consent_needed|outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]data_sharing_consent_needed|post_care_instructions[PERSONA:None]", + "[telehealth_setup]medication_instructions|missing|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|outdated|telehealth_consent_needed[PERSONA:Hard]", + "[telehealth_setup]medication_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]missing|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|post_care_instructions[PERSONA:None]", + "[telehealth_setup]outdated|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "[telehealth_setup]outdated|pre_surgery_instructions[PERSONA:Easy]", + "[telehealth_setup]outdated|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "[telehealth_setup]pre_surgery_instructions|telehealth_consent_needed[PERSONA:Hard]", + "[urgent_triage]high_fever|moderate_pain[PERSONA:Hard]", + "[urgent_triage]mild_fever|moderate_pain[PERSONA:Easy]", + "[urgent_triage]moderate_pain|very_high_fever[PERSONA:None]" + ] +} \ No newline at end of file diff --git a/data/tau2/domains/healthcare/tasks.json b/data/tau2/domains/healthcare/tasks.json new file mode 100644 index 00000000..60d44e47 --- /dev/null +++ b/data/tau2/domains/healthcare/tasks.json @@ -0,0 +1,10272 @@ +[ + { + "id": "[appointment_scheduling]doctor_available|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_calendar_conflicts|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|limited_availability|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]mild_fever|moderate_pain[PERSONA:Easy]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 100.2 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild fever", + "severity": "mild", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - mild fever" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]high_fever|moderate_pain[PERSONA:Hard]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 102.1 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "High fever with chills", + "severity": "moderate", + "duration": "2 days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - high fever" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]moderate_pain|very_high_fever[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 103.8 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Very high fever with severe chills", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - very high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_symptoms" + }, + "assert_value": true, + "message": "Agent transferred without assessing symptoms first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent identified severe symptoms but failed to transfer to clinical team" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]prediabetes|stage1[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|elevated[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|mild_hypoxemia[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|stage1[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|stage2[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia|stage1[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|mild_hypoxemia|prediabetes[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_5", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes|stage2[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|elevated|mild_hypoxemia[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_5", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia|stage2[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes|stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|outdated[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]pre_surgery_instructions|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|medication_instructions[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|outdated|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|medication_instructions|outdated[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_2", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|missing|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|missing|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack|wrong_medication_not_working[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp|wrong_dosage_confusion[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack|wrong_dosage_confusion[PERSONA:Hard]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp|wrong_medication_not_working[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 192, + "diastolic": 128 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild headache, feeling a bit off", + "severity": "mild", + "duration": "2 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports headache, needs BP check" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 52, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling dizzy and shaky, a bit confused", + "severity": "moderate", + "duration": "30 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - diabetic patient with dizziness" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 82, + "heart_rate": 92 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Short of breath when walking, tired", + "severity": "moderate", + "duration": "few days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_oxygen_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports breathing difficulty" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + } +] \ No newline at end of file diff --git a/data/tau2/domains/healthcare/tasks_full.json b/data/tau2/domains/healthcare/tasks_full.json new file mode 100644 index 00000000..b94d84c3 --- /dev/null +++ b/data/tau2/domains/healthcare/tasks_full.json @@ -0,0 +1,22150 @@ +[ + { + "id": "[prescription_refill]no_refills_remaining[PERSONA:None]", + "description": { + "purpose": "Test prescription refill request handling." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You need to refill your prescription medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's instructions. If they ask you to check medication bottles, use the check_medication_bottle tool. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling to request a prescription refill. Help them check prescription status and process refill if available, or guide them to contact their doctor if no refills remain.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_refills", + "arguments": { + "prescription_id": "rx_001", + "refills": 0 + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 0, + "prescribing_doctor": "Dr. Robert Williams", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_prescription_details" + }, + "assert_value": true, + "message": "Agent transferred without checking prescription details first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "request_prescription_refill" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to refill prescription that requires nurse escalation" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[prescription_refill]has_refills_available[PERSONA:Easy]", + "description": { + "purpose": "Test prescription refill request handling." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You need to refill your prescription medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's instructions. If they ask you to check medication bottles, use the check_medication_bottle tool. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling to request a prescription refill. Help them check prescription status and process refill if available, or guide them to contact their doctor if no refills remain.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_refills", + "arguments": { + "prescription_id": "rx_001", + "refills": 3 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Robert Williams", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_medication_bottle_1", + "requestor": "user", + "name": "check_medication_bottle", + "arguments": {} + }, + { + "action_id": "get_prescription_details_2", + "requestor": "assistant", + "name": "get_prescription_details", + "arguments": { + "prescription_id": "rx_001" + } + }, + { + "action_id": "verify_insurance_coverage_3", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "prescription_refill" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "request_prescription_refill_4", + "requestor": "assistant", + "name": "request_prescription_refill", + "arguments": { + "patient_id": "patient_001", + "prescription_id": "rx_001" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "expected_status": "active" + }, + "assert_value": true, + "message": "Prescription status should be active for successful refill" + }, + { + "env_type": "assistant", + "func_name": "assert_prescription_refills_remaining", + "arguments": { + "prescription_id": "rx_001", + "expected_count": 2 + }, + "assert_value": true, + "message": "Prescription should have 2 refills remaining after processing one refill" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION", + "ENV_ASSERTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|no_calendar_conflicts[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability|no_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|limited_availability[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|no_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_calendar_conflicts|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_calendar_conflicts|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|limited_availability|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|limited_availability|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|no_availability_preferred_times|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|insurance_verified|no_calendar_conflicts|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available|has_calendar_conflicts|insurance_verified|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|limited_availability|no_calendar_conflicts|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|routine_checkup[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|limited_availability|urgent_care_needed[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|routine_checkup[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified|no_availability_preferred_times|no_calendar_conflicts|urgent_care_needed[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts|insurance_verified|no_availability_preferred_times|urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + }, + { + "action_id": "verify_insurance_coverage_5", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_6", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_7", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_8", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]mild_fever[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 100.2 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild fever", + "severity": "mild", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - mild fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]high_fever[PERSONA:Easy]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 102.1 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "High fever with chills", + "severity": "moderate", + "duration": "2 days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]very_high_fever[PERSONA:Hard]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 103.8 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Very high fever with severe chills", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - very high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_symptoms" + }, + "assert_value": true, + "message": "Agent transferred without assessing symptoms first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent identified severe symptoms but failed to transfer to clinical team" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]moderate_pain[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]mild_fever|moderate_pain[PERSONA:Easy]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 100.2 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild fever", + "severity": "mild", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - mild fever" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]high_fever|moderate_pain[PERSONA:Hard]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 102.1 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "High fever with chills", + "severity": "moderate", + "duration": "2 days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - high fever" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]moderate_pain|very_high_fever[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 103.8 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Very high fever with severe chills", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - very high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_symptoms" + }, + "assert_value": true, + "message": "Agent transferred without assessing symptoms first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent identified severe symptoms but failed to transfer to clinical team" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 1 hypertension" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]stage2[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 2 hypertension" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_followup", + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - prediabetes" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Blood glucose monitoring and diabetes management" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_followup", + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - diabetes" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Blood glucose monitoring and diabetes management" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_spo2_followup", + "patient_id": "patient_001", + "reason": "Pending oxygen saturation monitoring - mild hypoxemia" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "11:00", + "reason": "Low oxygen saturation requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|elevated[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|mild_hypoxemia[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]prediabetes|stage1[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|stage1[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]prediabetes|stage2[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|stage2[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|stage2[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated|mild_hypoxemia|prediabetes[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_5", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|elevated|mild_hypoxemia[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_4", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_5", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes|stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia|stage1[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia|prediabetes|stage2[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_blood_glucose_4", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "11:00", + "reason": "Review home monitoring readings - blood pressure, glucose/diabetes, oxygen saturation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes|mild_hypoxemia|stage2[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "measure_oxygen_saturation_4", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "measure_blood_pressure_5", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "book_appointment_6", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Review home monitoring readings - oxygen saturation, blood pressure, glucose/diabetes" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]post_care_instructions[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]pre_surgery_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|telehealth_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]post_care_instructions|telehealth_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]pre_surgery_instructions|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_1", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|missing[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|medication_instructions[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|missing[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|outdated[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|missing|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions|outdated|telehealth_consent_needed[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|post_care_instructions|telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated|pre_surgery_instructions|telehealth_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "provide_consent_2", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|medication_instructions|missing[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_2", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|missing|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|missing|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|medication_instructions|outdated[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "acknowledge_instructions_1", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + }, + { + "action_id": "update_emergency_contact_2", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated|post_care_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed|outdated|pre_surgery_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + }, + { + "action_id": "update_emergency_contact_1", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + }, + { + "action_id": "acknowledge_instructions_2", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[test_results_access]normal[PERSONA:None]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_routine_followup", + "patient_id": "patient_001", + "reason": "Pending routine follow-up for normal test results" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": true, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "All test values within normal reference ranges", + "notes": "No abnormalities detected" + } + ], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_test_results_1", + "requestor": "assistant", + "name": "check_test_results", + "arguments": { + "patient_id": "patient_001", + "test_id": "test_001" + } + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "follow_up" + } + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine follow-up to review test results and annual wellness check" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No follow-up appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[test_results_access]abnormal_minor[PERSONA:Easy]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_followup", + "patient_id": "patient_001", + "reason": "Pending test results follow-up - minor abnormalities" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": true, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "Slightly elevated cholesterol (220 mg/dL) - recommend dietary modifications", + "notes": "Minor abnormality - follow-up in 3 months recommended" + } + ], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_test_results_1", + "requestor": "assistant", + "name": "check_test_results", + "arguments": { + "patient_id": "patient_001", + "test_id": "test_001" + } + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "follow_up" + } + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-30" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-30", + "time": "10:00", + "reason": "Follow-up for abnormal test results - discuss findings and treatment plan" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No follow-up appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[test_results_access]abnormal_critical[PERSONA:Hard]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_critical_transfer", + "patient_id": "patient_001", + "reason": "Pending test results - critical findings requiring immediate attention" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "test_results": [], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_test_results" + }, + "assert_value": true, + "message": "Agent transferred without checking test results first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + } + ] + }, + "evaluation_criteria": { + "actions": [], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "take_temperature" + }, + "assert_value": true, + "message": "Agent must verify patient's fever claim with objective temperature reading" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION" + ] + } + }, + { + "id": "[patient_mistake]wrong_medication_not_working[PERSONA:Hard]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]wrong_dosage_confusion[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack|wrong_medication_not_working[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack|wrong_dosage_confusion[PERSONA:Hard]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp|wrong_medication_not_working[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp|wrong_dosage_confusion[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 192, + "diastolic": 128 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild headache, feeling a bit off", + "severity": "mild", + "duration": "2 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports headache, needs BP check" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 52, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling dizzy and shaky, a bit confused", + "severity": "moderate", + "duration": "30 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - diabetic patient with dizziness" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 82, + "heart_rate": 92 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Short of breath when walking, tired", + "severity": "moderate", + "duration": "few days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_oxygen_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports breathing difficulty" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + } +] \ No newline at end of file diff --git a/data/tau2/domains/healthcare/tasks_small.json b/data/tau2/domains/healthcare/tasks_small.json new file mode 100644 index 00000000..a6854859 --- /dev/null +++ b/data/tau2/domains/healthcare/tasks_small.json @@ -0,0 +1,4405 @@ +[ + { + "id": "[prescription_refill]no_refills_remaining[PERSONA:None]", + "description": { + "purpose": "Test prescription refill request handling." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You need to refill your prescription medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's instructions. If they ask you to check medication bottles, use the check_medication_bottle tool. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling to request a prescription refill. Help them check prescription status and process refill if available, or guide them to contact their doctor if no refills remain.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_refills", + "arguments": { + "prescription_id": "rx_001", + "refills": 0 + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 0, + "prescribing_doctor": "Dr. Robert Williams", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_prescription_details" + }, + "assert_value": true, + "message": "Agent transferred without checking prescription details first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "request_prescription_refill" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to refill prescription that requires nurse escalation" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[prescription_refill]has_refills_available[PERSONA:Easy]", + "description": { + "purpose": "Test prescription refill request handling." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You need to refill your prescription medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's instructions. If they ask you to check medication bottles, use the check_medication_bottle tool. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling to request a prescription refill. Help them check prescription status and process refill if available, or guide them to contact their doctor if no refills remain.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_refills", + "arguments": { + "prescription_id": "rx_001", + "refills": 3 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Robert Williams", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_medication_bottle_1", + "requestor": "user", + "name": "check_medication_bottle", + "arguments": {} + }, + { + "action_id": "get_prescription_details_2", + "requestor": "assistant", + "name": "get_prescription_details", + "arguments": { + "prescription_id": "rx_001" + } + }, + { + "action_id": "verify_insurance_coverage_3", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "prescription_refill" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "request_prescription_refill_4", + "requestor": "assistant", + "name": "request_prescription_refill", + "arguments": { + "patient_id": "patient_001", + "prescription_id": "rx_001" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "expected_status": "active" + }, + "assert_value": true, + "message": "Prescription status should be active for successful refill" + }, + { + "env_type": "assistant", + "func_name": "assert_prescription_refills_remaining", + "arguments": { + "prescription_id": "rx_001", + "expected_count": 2 + }, + "assert_value": true, + "message": "Prescription should have 2 refills remaining after processing one refill" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION", + "ENV_ASSERTION" + ] + } + }, + { + "id": "[appointment_scheduling]doctor_available[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]limited_availability[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_availability_preferred_times[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]insurance_verified[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]no_calendar_conflicts[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]has_calendar_conflicts[PERSONA:Hard]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "10:00", + "available": false, + "reason": "Work meeting" + } + }, + { + "env_type": "user", + "func_name": "add_calendar_slot", + "arguments": { + "date": "2024-05-20", + "time": "14:00", + "available": true, + "reason": null + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]routine_checkup[PERSONA:None]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "routine_checkup" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[appointment_scheduling]urgent_care_needed[PERSONA:Easy]", + "description": { + "purpose": "Test appointment scheduling with various constraints." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to schedule a medical appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Follow the agent's guidance. Check your calendar and insurance when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "verify_insurance_coverage_1", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No new appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]mild_fever[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 100.2 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild fever", + "severity": "mild", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - mild fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]high_fever[PERSONA:Easy]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 102.1 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "High fever with chills", + "severity": "moderate", + "duration": "2 days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]very_high_fever[PERSONA:Hard]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 103.8 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Very high fever with severe chills", + "severity": "severe", + "duration": "6 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - very high fever" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_symptoms" + }, + "assert_value": true, + "message": "Agent transferred without assessing symptoms first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent identified severe symptoms but failed to transfer to clinical team" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[urgent_triage]moderate_pain[PERSONA:None]", + "description": { + "purpose": "Test urgent care triage with symptom assessment." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You are not feeling well and need medical attention.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "take_temperature_1", + "requestor": "user", + "name": "take_temperature", + "arguments": {} + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "urgent_care" + }, + "compare_args": [ + "patient_id" + ] + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No urgent appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]elevated[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 125, + "diastolic": 78 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]stage1[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 135, + "diastolic": 85 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 1 hypertension" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]stage2[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 148, + "diastolic": 94 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_followup", + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 2 hypertension" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_pressure_1", + "requestor": "user", + "name": "measure_blood_pressure", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]prediabetes[PERSONA:None]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 112, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_followup", + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - prediabetes" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Blood glucose monitoring and diabetes management" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]diabetes[PERSONA:Easy]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 145, + "measurement_time": "fasting" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_followup", + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - diabetes" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_blood_glucose_1", + "requestor": "user", + "name": "measure_blood_glucose", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Blood glucose monitoring and diabetes management" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[chronic_monitoring]mild_hypoxemia[PERSONA:Hard]", + "description": { + "purpose": "Test chronic condition monitoring with home measurements." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to discuss your home health monitoring readings.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You manage chronic health conditions.", + "task_instructions": "Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 92, + "heart_rate": 78 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_spo2_followup", + "patient_id": "patient_001", + "reason": "Pending oxygen saturation monitoring - mild hypoxemia" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "measure_oxygen_saturation_1", + "requestor": "user", + "name": "measure_oxygen_saturation", + "arguments": {} + }, + { + "action_id": "check_available_time_slots_2", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + } + }, + { + "action_id": "check_calendar_3", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_4", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "11:00", + "reason": "Low oxygen saturation requiring urgent evaluation" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No monitoring appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]telehealth_consent_needed[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]data_sharing_consent_needed[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "provide_consent", + "arguments": { + "consent_type": "telehealth" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "provide_consent_0", + "requestor": "user", + "name": "provide_consent", + "arguments": { + "consent_type": "data_sharing" + } + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]missing[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]outdated[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_emergency_contact", + "arguments": { + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "update_emergency_contact_0", + "requestor": "user", + "name": "update_emergency_contact", + "arguments": { + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family" + }, + "compare_args": [] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]medication_instructions[PERSONA:Easy]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]post_care_instructions[PERSONA:Hard]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[telehealth_setup]pre_surgery_instructions[PERSONA:None]", + "description": { + "purpose": "Test telehealth setup with consent and contact management." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to set up a telehealth appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "medication" + } + }, + { + "env_type": "user", + "func_name": "acknowledge_instructions", + "arguments": { + "instruction_type": "post_care" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "acknowledge_instructions_0", + "requestor": "user", + "name": "acknowledge_instructions", + "arguments": { + "instruction_type": "pre_surgery" + }, + "compare_args": [ + "instruction_type" + ] + } + ], + "env_assertions": [], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[test_results_access]normal[PERSONA:None]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_routine_followup", + "patient_id": "patient_001", + "reason": "Pending routine follow-up for normal test results" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": true, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "All test values within normal reference ranges", + "notes": "No abnormalities detected" + } + ], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_test_results_1", + "requestor": "assistant", + "name": "check_test_results", + "arguments": { + "patient_id": "patient_001", + "test_id": "test_001" + } + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "follow_up" + } + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-20" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine follow-up to review test results and annual wellness check" + }, + "compare_args": [ + "patient_id", + "doctor_id", + "appointment_type" + ] + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No follow-up appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[test_results_access]abnormal_minor[PERSONA:Easy]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_followup", + "patient_id": "patient_001", + "reason": "Pending test results follow-up - minor abnormalities" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": true, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "Slightly elevated cholesterol (220 mg/dL) - recommend dietary modifications", + "notes": "Minor abnormality - follow-up in 3 months recommended" + } + ], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "get_patient_details_0", + "requestor": "assistant", + "name": "get_patient_details", + "arguments": { + "full_name": "Sarah Johnson", + "date_of_birth": "1985-03-15" + } + }, + { + "action_id": "check_test_results_1", + "requestor": "assistant", + "name": "check_test_results", + "arguments": { + "patient_id": "patient_001", + "test_id": "test_001" + } + }, + { + "action_id": "verify_insurance_coverage_2", + "requestor": "assistant", + "name": "verify_insurance_coverage", + "arguments": { + "patient_id": "patient_001", + "procedure_type": "follow_up" + } + }, + { + "action_id": "check_available_time_slots_3", + "requestor": "assistant", + "name": "check_available_time_slots", + "arguments": { + "doctor_id": "doc_001", + "date": "2024-05-30" + }, + "compare_args": [ + "doctor_id" + ] + }, + { + "action_id": "check_calendar_4", + "requestor": "user", + "name": "check_calendar", + "arguments": {} + }, + { + "action_id": "book_appointment_5", + "requestor": "assistant", + "name": "book_appointment", + "arguments": { + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-30", + "time": "10:00", + "reason": "Follow-up for abnormal test results - discuss findings and treatment plan" + } + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_appointment_count_exceeds_baseline", + "arguments": {}, + "assert_value": true, + "message": "No follow-up appointment was created - appointment count did not exceed baseline" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION", + "ACTION" + ] + } + }, + { + "id": "[test_results_access]abnormal_critical[PERSONA:Hard]", + "description": { + "purpose": "Test accessing and reviewing medical test results." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You want to check your medical test results.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Ask about your test results. Use open_patient_portal to view results when available. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_test_results_critical_transfer", + "patient_id": "patient_001", + "reason": "Pending test results - critical findings requiring immediate attention" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "test_results": [], + "messages_count": 0, + "outstanding_balance": 0 + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_test_results" + }, + "assert_value": true, + "message": "Agent transferred without checking test results first" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly attempted to book appointment when transfer was required" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]chest_pain_vs_panic_attack[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.4 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]high_fever_vs_normal_temp[PERSONA:Easy]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_temperature", + "arguments": { + "temperature": 98.6 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours" + } + } + ] + }, + "evaluation_criteria": { + "actions": [], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "take_temperature" + }, + "assert_value": true, + "message": "Agent must verify patient's fever claim with objective temperature reading" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ENV_ASSERTION" + ] + } + }, + { + "id": "[patient_mistake]wrong_medication_not_working[PERSONA:Hard]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[patient_mistake]wrong_dosage_confusion[PERSONA:None]", + "description": { + "purpose": "Test handling of patient misinformation and verification workflows." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You have concerns about your symptoms or medication.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home.", + "task_instructions": "Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "assistant", + "func_name": "set_prescription_status", + "arguments": { + "prescription_id": "rx_001", + "status": "active" + } + }, + { + "env_type": "user", + "func_name": "add_medication_at_home", + "arguments": { + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "get_patient_details" + }, + "assert_value": true, + "message": "Agent must verify patient identity before addressing medical concerns" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "check_medication_bottle" + }, + "assert_value": true, + "message": "Agent must verify medication details before escalating confusion" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must escalate patient mistake to nurse for clinical review" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypertensive_crisis_standalone[PERSONA:None]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": null, + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_bp_monitor", + "arguments": { + "has_monitor": true, + "systolic": 192, + "diastolic": 128 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Mild headache, feeling a bit off", + "severity": "mild", + "duration": "2 hours" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_bp_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports headache, needs BP check" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]hypoglycemia_standalone[PERSONA:Easy]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAs a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology.\n\nYour health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments.\n\nIn interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_glucose_monitor", + "arguments": { + "has_monitor": true, + "glucose_reading": 52, + "measurement_time": "fasting" + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Feeling dizzy and shaky, a bit confused", + "severity": "moderate", + "duration": "30 minutes" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_glucose_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - diabetic patient with dizziness" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + }, + { + "id": "[critical_triage]severe_hypoxemia_standalone[PERSONA:Hard]", + "description": { + "purpose": "Test recognition of critical vital signs requiring immediate escalation." + }, + "user_scenario": { + "persona": "\nAt 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system.\n\nYour health literacy is limited - terms like \"copay,\" \"prior authorization,\" and \"formulary\" confuse you. You have trouble remembering medication names and often refer to them by what they're for (\"my blood pressure pill\"). You prefer when someone walks you through each step slowly.\n\nWhen dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly.\n", + "instructions": { + "domain": "healthcare", + "reason_for_call": "You're not feeling well and want to check if you need an appointment.", + "known_info": "You are Sarah Johnson, born on 1985-03-15, currently at home. You have home health monitoring devices.", + "task_instructions": "Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. If the tool call does not return updated information, you might need to perform another tool call to get the updated details. \nWhenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls.\nFor example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call.\nNever make up information, always ground your responses on the results of tool calls.\nIf you are unsure about whether an action is necessary, always ask the agent for clarification.\n" + } + }, + "ticket": "Patient Sarah Johnson (DOB: 1985-03-15) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP \u2265180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + "initial_state": { + "initialization_actions": [ + { + "env_type": "user", + "func_name": "set_user_info", + "arguments": { + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15" + } + }, + { + "env_type": "user", + "func_name": "set_user_location", + "arguments": { + "location": "home" + } + }, + { + "env_type": "user", + "func_name": "set_portal_info", + "arguments": { + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": false, + "messages_count": 0, + "outstanding_balance": 0 + } + }, + { + "env_type": "user", + "func_name": "set_pulse_oximeter", + "arguments": { + "has_monitor": true, + "spo2": 82, + "heart_rate": 92 + } + }, + { + "env_type": "user", + "func_name": "add_symptom", + "arguments": { + "description": "Short of breath when walking, tired", + "severity": "moderate", + "duration": "few days" + } + }, + { + "env_type": "assistant", + "func_name": "create_appointment_marker", + "arguments": { + "appointment_id": "pending_oxygen_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports breathing difficulty" + } + } + ] + }, + "evaluation_criteria": { + "actions": [ + { + "action_id": "transfer_to_nurse", + "requestor": "assistant", + "name": "transfer_to_nurse", + "arguments": {} + } + ], + "env_assertions": [ + { + "env_type": "assistant", + "func_name": "assert_tool_was_called", + "arguments": { + "tool_name": "transfer_to_nurse" + }, + "assert_value": true, + "message": "Agent must transfer to nurse for critical vital signs" + }, + { + "env_type": "assistant", + "func_name": "assert_tool_was_not_called", + "arguments": { + "tool_name": "book_appointment" + }, + "assert_value": true, + "message": "Agent incorrectly tried to book appointment for critical emergency" + } + ], + "communicate_info": null, + "nl_assertions": null, + "reward_basis": [ + "ACTION" + ] + } + } +] \ No newline at end of file diff --git a/data/tau2/domains/healthcare/user_db.json b/data/tau2/domains/healthcare/user_db.json new file mode 100644 index 00000000..9256e564 --- /dev/null +++ b/data/tau2/domains/healthcare/user_db.json @@ -0,0 +1,43 @@ +{ + "patient_device": { + "insurance_card": { + "provider": "BlueCross", + "policy_number": "BC123456789", + "group_number": "GRP001", + "member_name": "Default Patient", + "copay_info": "Primary Care: $20, Specialist: $35, Urgent Care: $50" + }, + "current_symptoms": [], + "current_temperature": null, + "medications_at_home": [], + "calendar_availability": [], + "portal_info": null, + "has_blood_pressure_monitor": true, + "latest_bp_reading": { + "systolic": 128, + "diastolic": 82 + }, + "has_glucose_meter": true, + "latest_glucose_reading": 105, + "glucose_measurement_time": "Fasting (8am)", + "has_pulse_oximeter": true, + "latest_spo2_reading": 98, + "latest_heart_rate": 72, + "current_pain": null, + "confirmed_appointments": [], + "consents_provided": [], + "acknowledged_instructions": [], + "notification_preferences": [], + "pharmacy_transfer_requests": [], + "uploaded_photos": [] + }, + "surroundings": { + "patient_id": "patient_001", + "full_name": "Default Patient", + "date_of_birth": "1990-01-01", + "location": "home", + "has_internet_access": true, + "payment_methods_available": ["credit_card"], + "emergency_contact": null + } +} diff --git a/src/tau2/data_model/message.py b/src/tau2/data_model/message.py index 479e876a..fa0bda98 100644 --- a/src/tau2/data_model/message.py +++ b/src/tau2/data_model/message.py @@ -58,6 +58,10 @@ class ToolCall(BaseModel): "assistant", description="The requestor of the tool call.", ) + compare_args: Optional[list[str]] = Field( + default=None, + description="List of argument keys to compare. If None, compare all. If empty list, compare only tool name.", + ) def __str__(self) -> str: lines = [f"ToolCall (from {self.requestor})"] diff --git a/src/tau2/domains/healthcare/README.md b/src/tau2/domains/healthcare/README.md new file mode 100644 index 00000000..12702d9f --- /dev/null +++ b/src/tau2/domains/healthcare/README.md @@ -0,0 +1,297 @@ +# Healthcare Domain + +The healthcare domain simulates a customer service environment where agents help patients with appointments, prescriptions, insurance verification, test results, and chronic condition monitoring. + +## Overview + +The healthcare domain tests agent capabilities in: + +- **Workflow compliance**: Identity verification first, insurance checks before booking +- **Clinical safety**: Specific thresholds determine when to escalate to medical staff (fever ≥103°F, BP ≥180/120, etc.) +- **Bidirectional coordination**: Agent and patient both have tools - agent guides, patient performs actions +- **Mixed evaluation**: Tasks check both correct outcomes (ENV_ASSERTION) and safe procedures (ACTION) +- **Patient personas**: Easy, None (neutral), and Hard personas with different health literacy levels + +## Domain at a Glance + +| Metric | Value | +|--------|-------| +| **Base Tasks** | 70 (152 full, 37 small) | +| **Intents** | 8 (appointment, prescription, monitoring, telehealth, test results, triage) | +| **Personas** | Easy (24), None (23), Hard (23) | +| **Evaluation** | Mixed (ENV + ACTION) for critical workflows | +| **Patient Records** | 3 with comprehensive medical history | +| **Agent Tools** | 18 (5 evaluated in tasks) | +| **User Tools** | 20 (evaluated via ENV_ASSERTION) | + +### Task Distribution by Intent + +**Base Set (tasks.json - 70 tasks):** +- `appointment_scheduling` (27) - Book/cancel/reschedule with insurance verification +- `telehealth_setup` (18) - Set up remote care with consent and instructions +- `chronic_monitoring` (15) - Monitor vitals for diabetes, hypertension, COPD +- `patient_mistake` (4) - Handle patient confusion gracefully +- `urgent_triage` (3) - Triage urgent symptoms appropriately (fever, pain, breathing) +- `critical_triage` (3) - Escalate critical conditions immediately (≥103°F, ≥180/120, <90% O2) + +**Full Set (tasks_full.json - 152 tasks):** +Includes all base tasks plus additional complexity variations: +- `appointment_scheduling` (71) - Extended scenarios with more edge cases +- `telehealth_setup` (35) - Additional consent and setup variations +- `chronic_monitoring` (23) - More vital sign combinations +- `patient_mistake` (8) - More confusion scenarios +- `urgent_triage` (7) - Additional symptom presentations +- `critical_triage` (3) - Same critical escalation tests +- `prescription_refill` (2) - Basic refill scenarios +- `test_results_access` (3) - Lab results review scenarios + +**Small Set (tasks_small.json - 37 tasks):** +Single-subtask tasks for quick evaluation, one per (intent × persona) combination. + +## Architecture + +The domain uses a bidirectional setup where both agent and patient have their own tools: + +``` +┌─────────────────────────────────────────────────────────┐ +│ HealthcareEnvironment │ +├────────────────────────┬────────────────────────────────┤ +│ Agent Side │ Patient Side │ +│ (HealthcareTools) │ (HealthcareUserTools) │ +├────────────────────────┼────────────────────────────────┤ +│ • get_patient_details │ • check_insurance_card │ +│ • verify_insurance │ • check_calendar │ +│ • book_appointment │ • measure_blood_pressure │ +│ • check_test_results │ • measure_blood_glucose │ +│ • transfer_to_nurse │ • provide_consent │ +│ • 20+ more tools │ • 15+ more tools │ +└────────────────────────┴────────────────────────────────┘ + ↓ ↓ + Agent performs Patient performs + system actions real-world actions +``` + +The agent can't directly access the patient's insurance card or vital signs. Instead, it must ask the patient to check these using user-side tools. This tests realistic coordination between agent requests and patient actions. + +## Getting Started + +### Quick Run + +```bash +# View domain info +tau2 domain healthcare + +# Test with 5 random tasks +tau2 run --domain healthcare \ + --agent-llm claude-sonnet-4-5-20250929 \ + --user-llm claude-sonnet-4-5-20250929 \ + --num-tasks 5 + +# Try tasks interactively (will prompt for domain and task selection) +tau2 play +``` + +## Example: Appointment Scheduling Flow + +**Scenario**: Patient needs routine checkup appointment + +**Required Workflow** (identity → insurance → availability → book): + +1. **Agent**: `get_patient_details("Sarah Johnson", "1985-03-15")` → Verify identity + - Returns: `patient_id="patient_001"` (needed for subsequent calls) + +2. **Agent asks**: "Can you check your insurance card?" + - **Patient**: `check_insurance_card()` → Returns: "BlueCross BlueShield, BC123456" + +3. **Agent**: `verify_insurance_coverage(patient_id="patient_001", procedure_type="routine_checkup")` → Check coverage + - Returns: Copay $20, covered + +4. **Agent**: `check_available_time_slots(doctor_id="doc_001", date="2024-05-20")` → Find slots + - Returns: Available times [09:00, 14:00, 16:00] + +5. **Agent asks**: "Can you check your calendar for May 20th at 2 PM?" + - **Patient**: `check_calendar()` → Returns: "May 20th is available" + +6. **Agent**: `book_appointment(patient_id="patient_001", doctor_id="doc_001", appointment_type="routine_checkup", date="2024-05-20", time="14:00", reason="Annual checkup")` → Book it + +**Evaluation**: +- **ENV_ASSERTION**: ✓ Appointment exists in database with correct details +- **ACTION**: ✓ Correct sequence (identity first, then insurance, then availability, then book) +- Both must pass for reward = 1.0 + +## Policies + +### Workflow Requirements + +**Multi-Step Pattern** (all workflows follow this hierarchy): +1. **Identity Verification** - Always verify patient identity first using `get_patient_details(full_name, date_of_birth)` +2. **Assessment** - Gather necessary information (insurance, symptoms, vitals) +3. **Verification** - Confirm availability, eligibility, or status +4. **Action** - Execute the requested operation only after all prerequisites met + +**Mandatory Sequences:** +- Appointment booking: identity → insurance → availability → book +- Prescription refills: identity → medication check → prescription verification → insurance → refill +- Chronic monitoring: identity → measure vitals → assess → schedule/transfer +- Telehealth setup: identity → consent → emergency contact → instructions + +### Clinical Safety Thresholds + +**Critical Values** (immediate transfer to nurse): +| Vital Sign | Transfer Threshold | Booking Range | +|------------|-------------------|---------------| +| **Fever** | ≥103°F | 100-102.9°F | +| **Pain** | ≥7/10 (severe) | 1-6/10 (mild-moderate) | +| **Blood Pressure** | ≥180/120 mmHg | 130-179/80-119 mmHg | +| **Blood Glucose** | <70 or >250 mg/dL | 100-250 mg/dL | +| **Oxygen Saturation** | <90% | 90-95% | + +**Decision Rule:** +- At or above transfer threshold → `transfer_to_nurse()` immediately +- Within booking range → `book_appointment()` for follow-up +- Normal readings → Routine follow-up scheduling + +### Communication Guidelines + +- State appointment details (date, time, doctor, specialty, copay) clearly after booking +- Explain next steps explicitly (when to arrive, what to bring) +- Transfer to nurse for clinical questions (interpreting test results, medication advice) +- Transfer to human agent for administrative issues (billing disputes, system errors) + +## Tools Overview + +### Agent-Side (HealthcareTools) + +**Identity & Records:** +- `get_patient_details(full_name, date_of_birth)` - Verify and retrieve patient record +- `get_chronic_conditions(patient_id)` - View patient's chronic conditions +- `get_vital_signs_history(patient_id)` - Review past vital readings + +**Appointments:** +- `book_appointment(...)` - Schedule appointment +- `cancel_appointment(appointment_id)` - Cancel existing appointment +- `check_available_time_slots(doctor_id, date)` - Find available slots +- `list_available_doctors(specialty)` - Find doctors by specialty + +**Insurance & Billing:** +- `verify_insurance_coverage(patient_id, procedure_type)` - Check coverage +- `calculate_cost(patient_id, appointment_type)` - Calculate copay + +**Clinical:** +- `check_test_results(patient_id)` - Access lab results +- `get_prescription_details(prescription_id)` - View prescription info +- `request_prescription_refill(patient_id, prescription_id)` - Refill prescription +- `transfer_to_nurse()` - Escalate to clinical staff + +### Patient-Side (HealthcareUserTools) + +**Information Access:** +- `check_insurance_card()` - View insurance provider and policy number +- `check_calendar()` - Check personal availability +- `check_medication_bottle()` - Read prescription number from bottle + +**Vital Measurements:** +- `measure_blood_pressure()` - Take BP reading +- `measure_blood_glucose()` - Check blood sugar +- `measure_oxygen_saturation()` - Measure O2 saturation +- `take_temperature()` - Measure temperature +- `check_symptoms()` - Describe current symptoms + +**Consent & Actions:** +- `provide_consent(consent_type)` - Give consent for telehealth/billing/data +- `acknowledge_instructions(instruction_type)` - Acknowledge medical instructions +- `update_emergency_contact()` - Update emergency contact info + + +## Adding Tasks + +To add new healthcare tasks: + +1. **Create task intent** in `src/tau2/domains/healthcare/tasks/_issues.py` + - Follow existing patterns in `appointment_issues.py`, `prescription_issues.py`, etc. + - Define `create__tasks()` function that returns list of tasks + +2. **Define evaluation functions** in `src/tau2/domains/healthcare/tasks/evaluation_functions.py` + - Add `is__fixed(env)` to check if issue is resolved + - Add `get__env_assertions(...)` if using ENV_ASSERTION checks + - Centralize reusable evaluation logic here + +3. **Set evaluation mode**: + - Use `["ENV_ASSERTION", "ACTION"]` for workflows requiring specific order (appointments, prescriptions) + - Use `["ENV_ASSERTION"]` for outcome-only checks (consent, emergency contact updates) + - Identity verification must be first action if using ACTION mode + - Insurance verification before booking/refilling if using ACTION mode + +4. **Register in task manager** in `src/tau2/domains/healthcare/tasks/create_tasks.py` + - Import your task creation function + - Add to the appropriate TaskManager + - Run `python -m src.tau2.domains.healthcare.tasks.create_tasks` to regenerate task files + +5. **Update splits** in `data/tau2/domains/healthcare/split_tasks.json` if adding to train/test sets + +## Testing + +Run the comprehensive test suite: + +```bash +# Test agent-side tools (39 tests) +pytest tests/test_domains/test_healthcare/test_tools_healthcare.py -v + +# Test patient-side tools (27 tests) +pytest tests/test_domains/test_healthcare/test_user_tools_healthcare.py -v + +# Test both (66 tests total) +pytest tests/test_domains/test_healthcare/ -v +``` + +All tests should pass (100% pass rate required). + +**Note**: You may see a pytest config warning about `asyncio_default_fixture_loop_scope` - this is harmless and can be ignored (the healthcare tests don't use async). + +## File Structure + +``` +src/tau2/domains/healthcare/ +├── __init__.py # Domain exports +├── README.md # This file +├── data_model.py # Patient, Doctor, Appointment, Prescription models +├── user_data_model.py # PatientDevice, PatientSurroundings +├── environment.py # HealthcareEnvironment + factory +├── tools.py # Agent-side tools (18 tools) +├── user_tools.py # Patient-side tools (20 tools) +├── utils.py # Path constants +└── tasks/ + ├── __init__.py + ├── const.py # Personas and tool grounding + ├── create_tasks.py # Task generation pipeline + ├── manager.py # TaskManager class + ├── utils.py # Task composition utilities + ├── evaluation_functions.py # Centralized evaluation logic + ├── appointment_issues.py # Appointment scheduling tasks + ├── prescription_issues.py # Prescription refill tasks + ├── chronic_monitoring_issues.py # Vital signs monitoring tasks + ├── telehealth_issues.py # Telehealth setup tasks + ├── test_results_issues.py # Lab results access tasks + ├── urgent_triage_issues.py # Urgent symptom triage tasks + ├── critical_triage_issues.py # Critical condition escalation + └── patient_mistake_issues.py # Patient confusion handling + +data/tau2/domains/healthcare/ +├── db.json # Patient database (3 patients) +├── user_db.json # Patient device state +├── policy.md # Agent policy (512 lines) +├── tasks.json # Main task set (70 tasks) +├── tasks_full.json # All tasks (152 tasks) +├── tasks_small.json # Single-intent tasks (37 tasks) +└── split_tasks.json # Train/dev/test splits + +tests/test_domains/test_healthcare/ +├── test_tools_healthcare.py # Agent tool tests (39 tests) +└── test_user_tools_healthcare.py # Patient tool tests (27 tests) +``` + +## Additional Documentation + +- **Agent Policy**: Full policy with clinical thresholds and workflow requirements at `data/tau2/domains/healthcare/policy.md` +- **Data Models**: Detailed schema documentation in `src/tau2/domains/healthcare/data_model.py` +- **Task Generation**: Implementation details in `src/tau2/domains/healthcare/tasks/` \ No newline at end of file diff --git a/src/tau2/domains/healthcare/__init__.py b/src/tau2/domains/healthcare/__init__.py new file mode 100644 index 00000000..fca0cd45 --- /dev/null +++ b/src/tau2/domains/healthcare/__init__.py @@ -0,0 +1 @@ +# Copyright Sierra diff --git a/src/tau2/domains/healthcare/data_model.py b/src/tau2/domains/healthcare/data_model.py new file mode 100644 index 00000000..0ca70752 --- /dev/null +++ b/src/tau2/domains/healthcare/data_model.py @@ -0,0 +1,384 @@ +from typing import Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, Field + +from tau2.domains.healthcare.utils import HEALTHCARE_DB_PATH +from tau2.environment.db import DB + +# Type definitions +AppointmentType = Literal["routine_checkup", "follow_up", "urgent_care", "specialist"] +AppointmentStatus = Literal["scheduled", "completed", "cancelled", "no_show"] +InsuranceProvider = Literal[ + "BlueCross", "Aetna", "UnitedHealth", "Medicare", "Medicaid", "SelfPay" +] +PrescriptionStatus = Literal["active", "expired", "refill_needed", "discontinued"] +TestResultStatus = Literal["pending", "ready", "reviewed"] +ConditionSeverity = Literal["mild", "moderate", "severe"] +MedicationRoute = Literal["oral", "injection", "topical", "inhaled"] +AllergySeverity = Literal["mild", "moderate", "severe", "life_threatening"] +LabResultStatus = Literal["pending", "resulted", "reviewed"] +Priority = Literal["routine", "urgent", "stat"] + + +class Name(BaseModel): + """Patient or doctor name.""" + + first_name: str = Field(description="First name") + last_name: str = Field(description="Last name") + + +class InsurancePlan(BaseModel): + """Insurance plan information.""" + + provider: InsuranceProvider = Field(description="Insurance provider name") + policy_number: str = Field(description="Insurance policy number") + group_number: str = Field(description="Insurance group number") + copay_amount: int = Field(description="Standard copay amount in dollars") + coverage_details: str = Field(description="Brief description of coverage") + + +class ContactInfo(BaseModel): + """Contact information.""" + + phone: str = Field(description="Phone number") + email: str = Field(description="Email address") + address: str = Field(description="Full address") + + +class MedicalCondition(BaseModel): + """Chronic medical condition with diagnostic information.""" + + condition_name: str = Field(description="Name of the medical condition") + icd10_code: str = Field(description="ICD-10 diagnostic code") + diagnosed_date: str = Field(description="Date diagnosed in YYYY-MM-DD format") + severity: ConditionSeverity = Field(description="Severity of the condition") + controlled: bool = Field(description="Whether the condition is well-controlled") + requires_monitoring: bool = Field( + description="Whether regular monitoring is required" + ) + + +class Medication(BaseModel): + """Current medication with detailed information.""" + + medication_id: str = Field(description="Unique medication identifier") + name: str = Field(description="Brand or generic name of medication") + generic_name: str = Field(description="Generic pharmaceutical name") + dosage: str = Field(description="Dosage amount (e.g., '10mg', '500mg')") + frequency: str = Field( + description="How often taken (e.g., 'once daily', 'twice daily')" + ) + route: MedicationRoute = Field(description="Route of administration") + prescribed_date: str = Field(description="Date prescribed in YYYY-MM-DD format") + indication: str = Field(description="Medical reason for prescription") + interactions: List[str] = Field( + default_factory=list, description="List of medications this interacts with" + ) + side_effects: List[str] = Field( + default_factory=list, description="Common side effects" + ) + + +class Allergy(BaseModel): + """Allergy with reaction details and severity.""" + + allergen: str = Field(description="Substance causing allergic reaction") + reaction_type: str = Field( + description="Type of reaction (e.g., 'rash', 'anaphylaxis', 'hives')" + ) + severity: AllergySeverity = Field(description="Severity of allergic reaction") + onset_date: Optional[str] = Field( + default=None, description="When allergy was first identified" + ) + + +class VitalSigns(BaseModel): + """Vital signs measurement with timestamp.""" + + timestamp: str = Field(description="When vitals were measured (ISO format)") + blood_pressure_systolic: Optional[int] = Field( + default=None, description="Systolic BP in mmHg" + ) + blood_pressure_diastolic: Optional[int] = Field( + default=None, description="Diastolic BP in mmHg" + ) + heart_rate: Optional[int] = Field( + default=None, description="Heart rate in beats per minute" + ) + temperature: Optional[float] = Field( + default=None, description="Body temperature in °F" + ) + respiratory_rate: Optional[int] = Field( + default=None, description="Breaths per minute" + ) + oxygen_saturation: Optional[int] = Field( + default=None, description="SpO2 percentage" + ) + weight: Optional[float] = Field(default=None, description="Weight in kg") + height: Optional[float] = Field(default=None, description="Height in cm") + + +class LabResult(BaseModel): + """Laboratory test result with detailed values.""" + + test_id: str = Field(description="Unique test identifier") + patient_id: str = Field(description="Patient ID") + test_type: str = Field( + description="Type of test (e.g., 'CBC', 'HbA1c', 'Lipid Panel')" + ) + test_date: str = Field(description="Date test was performed in YYYY-MM-DD format") + results: Dict[str, Dict[str, Union[float, int, str]]] = Field( + description="Test results with values, units, and flags" + ) + ordering_doctor: str = Field(description="Doctor who ordered the test") + status: LabResultStatus = Field(description="Status of the lab result") + critical: bool = Field( + default=False, + description="Whether result is critical and requires immediate action", + ) + + +class LabOrder(BaseModel): + """Laboratory test order.""" + + order_id: str = Field(description="Unique order identifier") + patient_id: str = Field(description="Patient ID") + test_type: str = Field(description="Type of test ordered") + priority: Priority = Field(description="Priority level") + clinical_indication: str = Field(description="Medical reason for ordering test") + ordered_date: str = Field(description="Date ordered in YYYY-MM-DD format") + status: Literal["pending", "completed", "cancelled"] = Field( + description="Status of the order" + ) + + +class EmergencyTransfer(BaseModel): + """Emergency transfer record.""" + + patient_id: str = Field(description="Patient ID") + reason: str = Field(description="Reason for emergency transfer") + symptoms: List[str] = Field(description="List of concerning symptoms") + timestamp: str = Field(description="When transfer was initiated (ISO format)") + vital_signs: Optional[VitalSigns] = Field( + default=None, description="Latest vital signs" + ) + current_medications: List[str] = Field( + default_factory=list, description="Medications patient is currently taking" + ) + + +class Patient(BaseModel): + """Patient information in the system with comprehensive medical record.""" + + patient_id: str = Field(description="Unique patient identifier") + name: Name = Field(description="Patient name") + date_of_birth: str = Field(description="Date of birth in YYYY-MM-DD format") + contact: ContactInfo = Field(description="Contact information") + insurance: InsurancePlan = Field(description="Insurance plan") + + # Enriched medical data + chronic_conditions: List[MedicalCondition] = Field( + default_factory=list, description="Detailed chronic medical conditions" + ) + current_medications_detailed: List[Medication] = Field( + default_factory=list, + description="Detailed current medications with interactions and side effects", + ) + allergies_detailed: List[Allergy] = Field( + default_factory=list, + description="Detailed allergies with severity and reaction type", + ) + vital_signs_history: List[VitalSigns] = Field( + default_factory=list, description="Historical vital signs measurements" + ) + + # References to other records + appointment_ids: List[str] = Field( + default_factory=list, description="List of appointment IDs for this patient" + ) + prescription_ids: List[str] = Field( + default_factory=list, description="List of prescription IDs for this patient" + ) + lab_result_ids: List[str] = Field( + default_factory=list, description="List of lab result IDs for this patient" + ) + + # Clinical tracking + last_consultation_date: Optional[str] = Field( + default=None, description="Date of last consultation" + ) + last_hba1c_date: Optional[str] = Field( + default=None, description="Date of last HbA1c test (for diabetics)" + ) + last_lipid_panel_date: Optional[str] = Field( + default=None, + description="Date of last lipid panel (for cardiovascular monitoring)", + ) + + # Risk flags + high_risk_conditions: List[str] = Field( + default_factory=list, + description="List of high-risk conditions requiring special attention", + ) + needs_urgent_follow_up: bool = Field( + default=False, description="Whether patient requires urgent follow-up" + ) + + +class Doctor(BaseModel): + """Doctor information in the system.""" + + doctor_id: str = Field(description="Unique doctor identifier") + name: Name = Field(description="Doctor name") + specialty: str = Field(description="Medical specialty") + available_days: List[str] = Field( + description="Days of the week available (e.g., ['Monday', 'Wednesday', 'Friday'])" + ) + available_times: List[str] = Field( + description="Available time slots (e.g., ['09:00', '10:00', '14:00'])" + ) + + +class Appointment(BaseModel): + """Appointment information.""" + + appointment_id: str = Field(description="Unique appointment identifier") + patient_id: str = Field(description="Patient ID") + doctor_id: str = Field(description="Doctor ID") + appointment_type: AppointmentType = Field(description="Type of appointment") + date: str = Field(description="Appointment date in YYYY-MM-DD format") + time: str = Field(description="Appointment time in HH:MM format (24-hour)") + status: AppointmentStatus = Field(description="Current status of the appointment") + reason: str = Field(description="Reason for visit") + notes: Optional[str] = Field(default=None, description="Additional notes") + created_at: str = Field(description="When appointment was created (ISO format)") + cost: int = Field(description="Appointment cost in dollars (before insurance)") + + +class Prescription(BaseModel): + """Prescription information.""" + + prescription_id: str = Field(description="Unique prescription identifier") + patient_id: str = Field(description="Patient ID") + doctor_id: str = Field(description="Prescribing doctor ID") + medication_name: str = Field(description="Name of the medication") + dosage: str = Field(description="Dosage instructions (e.g., '10mg once daily')") + quantity: int = Field(description="Quantity prescribed") + refills_remaining: int = Field(description="Number of refills remaining") + status: PrescriptionStatus = Field(description="Current prescription status") + prescribed_date: str = Field(description="Date prescribed in YYYY-MM-DD format") + expiration_date: str = Field(description="Expiration date in YYYY-MM-DD format") + + +class TestResult(BaseModel): + """Medical test result information.""" + + test_id: str = Field(description="Unique test identifier") + patient_id: str = Field(description="Patient ID") + test_name: str = Field(description="Name of the test") + test_date: str = Field(description="Date test was performed in YYYY-MM-DD format") + status: TestResultStatus = Field(description="Status of the test result") + result: Optional[str] = Field( + default=None, description="Test result details (only if ready)" + ) + notes: Optional[str] = Field( + default=None, description="Doctor's notes on the result" + ) + + +class Payment(BaseModel): + """Payment transaction.""" + + payment_id: str = Field(description="Unique payment identifier") + patient_id: str = Field(description="Patient ID") + amount: int = Field(description="Payment amount in dollars") + payment_method: Literal["credit_card", "debit_card", "insurance", "cash"] = Field( + description="Method of payment" + ) + date: str = Field(description="Payment date in YYYY-MM-DD format") + description: str = Field(description="What the payment was for") + + +class HealthcareDB(DB): + """ + Main database for the healthcare domain. + Contains all patients, doctors, appointments, prescriptions, test results, and lab data. + """ + + patients: Dict[str, Patient] = Field( + default_factory=dict, description="Dictionary of patients keyed by patient_id" + ) + doctors: Dict[str, Doctor] = Field( + default_factory=dict, description="Dictionary of doctors keyed by doctor_id" + ) + appointments: Dict[str, Appointment] = Field( + default_factory=dict, + description="Dictionary of appointments keyed by appointment_id", + ) + prescriptions: Dict[str, Prescription] = Field( + default_factory=dict, + description="Dictionary of prescriptions keyed by prescription_id", + ) + test_results: Dict[str, TestResult] = Field( + default_factory=dict, description="Dictionary of test results keyed by test_id" + ) + payments: Dict[str, Payment] = Field( + default_factory=dict, description="Dictionary of payments keyed by payment_id" + ) + # New medical data collections + lab_results: Dict[str, LabResult] = Field( + default_factory=dict, description="Dictionary of lab results keyed by test_id" + ) + lab_orders: Dict[str, LabOrder] = Field( + default_factory=dict, description="Dictionary of lab orders keyed by order_id" + ) + emergency_transfers: Dict[str, EmergencyTransfer] = Field( + default_factory=dict, + description="Dictionary of emergency transfers keyed by patient_id", + ) + + tool_call_history: List[str] = Field( + default_factory=list, + description="History of assistant tool calls made during the conversation (tool names only)", + ) + + @classmethod + def get_db_path(cls): + """Get the default database path.""" + return HEALTHCARE_DB_PATH + + +# Export types for convenience +__all__ = [ + # Main models + "Patient", + "Doctor", + "Appointment", + "Prescription", + "TestResult", + "Payment", + "HealthcareDB", + # New medical models + "MedicalCondition", + "Medication", + "Allergy", + "VitalSigns", + "LabResult", + "LabOrder", + "EmergencyTransfer", + # Type definitions + "AppointmentType", + "AppointmentStatus", + "InsuranceProvider", + "PrescriptionStatus", + "TestResultStatus", + "ConditionSeverity", + "MedicationRoute", + "AllergySeverity", + "LabResultStatus", + "Priority", + # Supporting classes + "Name", + "InsurancePlan", + "ContactInfo", +] diff --git a/src/tau2/domains/healthcare/environment.py b/src/tau2/domains/healthcare/environment.py new file mode 100644 index 00000000..c55c7d6c --- /dev/null +++ b/src/tau2/domains/healthcare/environment.py @@ -0,0 +1,187 @@ +# Copyright Sierra +from pathlib import Path +from typing import Optional, cast + +from tau2.data_model.tasks import Task +from tau2.domains.healthcare.data_model import HealthcareDB +from tau2.domains.healthcare.tools import HealthcareTools +from tau2.domains.healthcare.user_data_model import HealthcareUserDB +from tau2.domains.healthcare.user_tools import HealthcareUserTools +from tau2.domains.healthcare.utils import ( + HEALTHCARE_DB_PATH, + HEALTHCARE_POLICY_PATH, + HEALTHCARE_TASK_SET_PATH, + HEALTHCARE_TASK_SET_FULL_PATH, + HEALTHCARE_TASK_SET_SMALL_PATH, + HEALTHCARE_USER_DB_PATH, +) +from tau2.environment.environment import Environment +from tau2.utils import load_file + + +class HealthcareEnvironment(Environment): + """ + Healthcare environment with bidirectional tool support. + Syncs agent-side and patient-side information. + """ + + tools: HealthcareTools + user_tools: HealthcareUserTools + + def __init__( + self, + domain_name: str, + policy: str, + tools: HealthcareTools, + user_tools: HealthcareUserTools, + ): + super().__init__(domain_name, policy, tools, user_tools) + + def make_tool_call(self, tool_name: str, requestor: str = "assistant", **kwargs): + """Override to track assistant tool calls in real-time for behavioral assertions.""" + if requestor == "assistant": + self.tools.db.tool_call_history.append(tool_name) + + return super().make_tool_call(tool_name, requestor, **kwargs) + + def set_state(self, initialization_data, initialization_actions, message_history): + """Override to track tool calls for behavioral assertions.""" + super().set_state(initialization_data, initialization_actions, message_history) + + from tau2.data_model.message import AssistantMessage, UserMessage + + tool_calls = [] + for message in message_history: + if ( + isinstance(message, (AssistantMessage, UserMessage)) + and message.is_tool_call() + ): + for tc in message.tool_calls: + if tc.requestor == "assistant": + tool_calls.append(tc.name) + + self.tools.db.tool_call_history = tool_calls + + def sync_tools(self): + """Sync agent and patient tool state.""" + patient_id = self.user_tools.surroundings.patient_id + + if patient_id not in self.tools.db.patients: + return + + patient = self.tools.db.patients[patient_id] + + if self.user_tools.device.portal_info: + portal = self.user_tools.device.portal_info + + upcoming_apts = [] + for apt_id in patient.appointment_ids: + if apt_id in self.tools.db.appointments: + apt = self.tools.db.appointments[apt_id] + if apt.status == "scheduled": + upcoming_apts.append( + f"{apt.date} at {apt.time} - {apt.appointment_type} with Dr. {self.tools.db.doctors[apt.doctor_id].name.last_name}" + ) + + portal.upcoming_appointments = upcoming_apts[:3] + + total_balance = 0 + for payment in self.tools.db.payments.values(): + if payment.patient_id == patient_id: + pass + portal.outstanding_balance = total_balance + + +def get_environment( + db: Optional[HealthcareDB] = None, + user_db: Optional[HealthcareUserDB] = None, + solo_mode: bool = False, +) -> HealthcareEnvironment: + """ + Create a healthcare environment instance. + + Args: + db: Optional agent-side database. If None, loads from default path. + user_db: Optional user-side database. If None, loads from default path. + solo_mode: Whether to run in solo mode (no user interaction) + + Returns: + Configured HealthcareEnvironment instance + """ + if db is None: + db = cast(HealthcareDB, HealthcareDB.load(str(HEALTHCARE_DB_PATH))) + + tools = HealthcareTools(db) + + if not solo_mode: + if user_db is None: + user_db = cast( + HealthcareUserDB, HealthcareUserDB.load(str(HEALTHCARE_USER_DB_PATH)) + ) + user_tools = HealthcareUserTools(user_db) + else: + raise ValueError("Healthcare domain does not yet support solo mode") + + with open(HEALTHCARE_POLICY_PATH, "r") as fp: + policy = fp.read() + + env = HealthcareEnvironment( + domain_name="healthcare", + policy=policy, + tools=tools, + user_tools=user_tools, + ) + + return env + + +def get_tasks(task_split_name: Optional[str] = "base") -> list[Task]: + """ + Load healthcare tasks from the task file. + + Args: + task_split_name: Optional task split name. Supported splits: "base", "train", "test". + + Returns: + List of Task objects filtered by the specified split + """ + tasks = load_file(HEALTHCARE_TASK_SET_PATH) + tasks = [Task.model_validate(task) for task in tasks] + + if task_split_name is None: + return tasks + + task_splits = get_tasks_split() + if task_split_name not in task_splits: + raise ValueError( + f"Invalid task split name: {task_split_name}. Valid splits are: {list(task_splits.keys())}" + ) + + tasks = [task for task in tasks if task.id in task_splits[task_split_name]] + return tasks + + +def get_tasks_split() -> dict[str, list[str]]: + """ + Load task split definitions from split_tasks.json. + + Returns: + Dictionary mapping split names ("train", "test", "base") to lists of task IDs + """ + split_file = ( + Path(HEALTHCARE_TASK_SET_PATH).parent + / f"split_{Path(HEALTHCARE_TASK_SET_PATH).stem}.json" + ) + return load_file(split_file) + + +def get_tasks_full(task_split_name: Optional[str] = None) -> list[Task]: + """Load the full healthcare task set from tasks_full.json.""" + tasks = load_file(HEALTHCARE_TASK_SET_FULL_PATH) + return [Task.model_validate(task) for task in tasks] + + +def get_tasks_small(task_split_name: Optional[str] = None) -> list[Task]: + """Load the small healthcare task set from tasks_small.json.""" + tasks = load_file(HEALTHCARE_TASK_SET_SMALL_PATH) + return [Task.model_validate(task) for task in tasks] diff --git a/src/tau2/domains/healthcare/tasks/__init__.py b/src/tau2/domains/healthcare/tasks/__init__.py new file mode 100644 index 00000000..70bf21f2 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/__init__.py @@ -0,0 +1 @@ +"""Healthcare task generation module.""" diff --git a/src/tau2/domains/healthcare/tasks/appointment_issues.py b/src/tau2/domains/healthcare/tasks/appointment_issues.py new file mode 100644 index 00000000..55aaaf11 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/appointment_issues.py @@ -0,0 +1,439 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### Init Functions + + +def init_doctor_available(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Doctor has multiple time slots available.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - doctor available", + }, + ) + ] + + +def init_limited_availability(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Doctor has limited availability.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - limited availability", + }, + ) + ] + + +def init_no_availability_preferred_times( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Doctor has no availability during patient's preferred times.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - preferred times unavailable", + }, + ) + ] + + +### Fix Functions + + +def fix_book_available_appointment(env: HealthcareEnvironment) -> list[ToolCall]: + """Book appointment in available slot.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={ + "patient_id": "patient_001", + "procedure_type": "routine_checkup", + }, + compare_args=["patient_id"], + ), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + compare_args=["doctor_id"], + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "routine_checkup", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine checkup appointment", + }, + compare_args=["patient_id", "doctor_id", "appointment_type"], + ), + ] + + +def init_insurance_verified( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Insurance is on file and verified.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance verified", + }, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_patient_has_insurance", + arguments={"patient_id": "patient_001", "expected": True}, + assert_value=True, + ), + ] + + +def init_insurance_not_on_file(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Insurance information is missing from patient record.""" + if "patient_001" in env.tools.db.patients: + patient = env.tools.db.patients["patient_001"] + from tau2.domains.healthcare.data_model import InsurancePlan + + patient.insurance = InsurancePlan( + provider="SelfPay", + policy_number="", + group_number="", + copay_amount=0, + coverage_details="No insurance on file", + ) + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_insurance_not_on_file", + "patient_id": "patient_001", + "reason": "Pending booking request - insurance not on file", + }, + ) + ] + + +def init_insurance_coverage_limited( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Insurance has limited coverage for requested service.""" + if "patient_001" in env.tools.db.patients: + patient = env.tools.db.patients["patient_001"] + patient.insurance.coverage_details = ( + "Limited coverage - specialist visits require referral" + ) + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_insurance_coverage_limited", + "patient_id": "patient_001", + "reason": "Pending booking request - limited insurance coverage", + }, + ) + ] + + +def init_no_calendar_conflicts(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Patient has no calendar conflicts.""" + env.user_tools.device.calendar_availability = [] + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - no calendar conflicts", + }, + ) + ] + + +def init_has_calendar_conflicts(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Patient has conflicts on some proposed dates.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_calendar_slot", + arguments={ + "date": "2024-05-20", + "time": "10:00", + "available": False, + "reason": "Work meeting", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_calendar_slot", + arguments={ + "date": "2024-05-20", + "time": "14:00", + "available": True, + "reason": None, + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - has calendar conflicts", + }, + ), + ] + + +def init_routine_checkup(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Simple routine checkup appointment.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_book_appointment", + "patient_id": "patient_001", + "reason": "Pending booking request - routine checkup", + }, + ) + ] + + +def init_specialist_referral_needed( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Appointment requires specialist referral.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Persistent heart palpitations requiring cardiology evaluation", + "severity": "moderate", + "duration": "2 weeks", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_specialist_referral_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - specialist referral needed", + }, + ), + ] + + +def init_urgent_care_needed(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Urgent care appointment needed due to severity.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Severe abdominal pain", + "severity": "severe", + "duration": "6 hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_care_needed", + "patient_id": "patient_001", + "reason": "Pending booking request - urgent care needed", + }, + ), + ] + + +def fix_urgent_care_appointment(env: HealthcareEnvironment) -> list[ToolCall]: + """Book urgent care appointment.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={"patient_id": "patient_001", "procedure_type": "urgent_care"}, + compare_args=["patient_id"], + ), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + compare_args=["doctor_id"], + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "15:00", + "reason": "Urgent care - severe symptoms requiring immediate evaluation", + }, + compare_args=["patient_id", "doctor_id", "appointment_type"], + ), + ] + + +### Base Tasks + +doctor_available_task = BaseTask( + name="doctor_available", + description="Doctor has multiple available time slots", + init_funcs=[init_doctor_available], + fix_funcs=[fix_book_available_appointment], +) + +limited_availability_task = BaseTask( + name="limited_availability", + description="Doctor has limited availability", + init_funcs=[init_limited_availability], + fix_funcs=[fix_book_available_appointment], +) + +no_availability_preferred_times_task = BaseTask( + name="no_availability_preferred_times", + description="No availability during patient's preferred times", + init_funcs=[init_no_availability_preferred_times], + fix_funcs=[fix_book_available_appointment], +) + +insurance_verified_task = BaseTask( + name="insurance_verified", + description="Insurance on file and verified", + init_funcs=[init_insurance_verified], + fix_funcs=[fix_book_available_appointment], +) + +insurance_not_on_file_task = BaseTask( + name="insurance_not_on_file", + description="Insurance information missing from record", + init_funcs=[init_insurance_not_on_file], + fix_funcs=[None], +) + +insurance_coverage_limited_task = BaseTask( + name="insurance_coverage_limited", + description="Insurance has limited coverage for service", + init_funcs=[init_insurance_coverage_limited], + fix_funcs=[None], +) + +no_calendar_conflicts_task = BaseTask( + name="no_calendar_conflicts", + description="No calendar conflicts", + init_funcs=[init_no_calendar_conflicts], + fix_funcs=[fix_book_available_appointment], +) + +has_calendar_conflicts_task = BaseTask( + name="has_calendar_conflicts", + description="Patient has calendar conflicts on some dates", + init_funcs=[init_has_calendar_conflicts], + fix_funcs=[fix_book_available_appointment], +) + +routine_checkup_task = BaseTask( + name="routine_checkup", + description="Simple routine checkup", + init_funcs=[init_routine_checkup], + fix_funcs=[fix_book_available_appointment], +) + +specialist_referral_needed_task = BaseTask( + name="specialist_referral_needed", + description="Requires specialist referral", + init_funcs=[init_specialist_referral_needed], + fix_funcs=[None], +) + +urgent_care_needed_task = BaseTask( + name="urgent_care_needed", + description="Urgent care appointment needed", + init_funcs=[init_urgent_care_needed], + fix_funcs=[fix_urgent_care_appointment], +) + + +### SelectionSets + +doctor_availability_issues = SelectionSet( + tasks=[ + doctor_available_task, + limited_availability_task, + no_availability_preferred_times_task, + ] +) + +insurance_verification_issues = SelectionSet( + tasks=[ + insurance_verified_task, + ] +) + +calendar_conflict_issues = SelectionSet( + tasks=[ + no_calendar_conflicts_task, + has_calendar_conflicts_task, + ] +) + +appointment_type_complexity = SelectionSet( + tasks=[ + routine_checkup_task, + urgent_care_needed_task, + ] +) + +appointment_scheduling_selection_sets = [ + doctor_availability_issues, + insurance_verification_issues, + calendar_conflict_issues, + appointment_type_complexity, +] diff --git a/src/tau2/domains/healthcare/tasks/chronic_monitoring_issues.py b/src/tau2/domains/healthcare/tasks/chronic_monitoring_issues.py new file mode 100644 index 00000000..205b3ffa --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/chronic_monitoring_issues.py @@ -0,0 +1,790 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet, ComposedTask + + +### Init Functions + + +def init_bp_normal(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood pressure in normal range <120/80.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={"has_monitor": True, "systolic": 118, "diastolic": 78}, + ) + ] + + +def init_bp_elevated(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood pressure elevated 120-129/<80.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={"has_monitor": True, "systolic": 125, "diastolic": 78}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_bp_followup", # Shared for all BP monitoring appointments + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up", + }, + ), + ] + + +def init_bp_stage1(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood pressure Stage 1 Hypertension 130-139/80-89.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={"has_monitor": True, "systolic": 135, "diastolic": 85}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_bp_followup", # Shared for all BP monitoring appointments + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 1 hypertension", + }, + ), + ] + + +def init_bp_stage2(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood pressure Stage 2 Hypertension e140/90.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={"has_monitor": True, "systolic": 148, "diastolic": 94}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_bp_followup", # Shared for all BP monitoring appointments + "patient_id": "patient_001", + "reason": "Pending BP monitoring follow-up - Stage 2 hypertension", + }, + ), + ] + + +def init_bp_hypertensive_crisis(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood pressure Hypertensive Crisis e180/120.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={"has_monitor": True, "systolic": 185, "diastolic": 122}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_chronic_emergency_transfer", # For emergency transfers + "patient_id": "patient_001", + "reason": "Pending emergency transfer - hypertensive crisis", + }, + ), + ] + + +### Fix Functions + + +def fix_bp_elevated_monitor(env: HealthcareEnvironment) -> list[ToolCall]: + """Schedule follow-up for elevated BP.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_pressure", arguments={}), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "15:00", + "reason": "Follow-up for elevated blood pressure monitoring", + }, + ), + ] + + +def fix_bp_crisis_emergency(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for hypertensive crisis.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_pressure", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def fix_bp_normal_routine(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient and blood pressure, inform all is well.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_pressure", arguments={}), + ] + + +def fix_glucose_normal_routine(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient and glucose, inform all is well.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_glucose", arguments={}), + ] + + +def fix_spo2_normal_routine(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient and oxygen saturation, inform all is well.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_oxygen_saturation", arguments={}), + ] + + +def init_glucose_normal(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood glucose normal 70-100 mg/dL fasting.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_glucose_monitor", + arguments={ + "has_monitor": True, + "glucose_reading": 92, + "measurement_time": "fasting", + }, + ) + ] + + +def init_glucose_prediabetes(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood glucose prediabetes range 100-125 mg/dL.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_glucose_monitor", + arguments={ + "has_monitor": True, + "glucose_reading": 112, + "measurement_time": "fasting", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_glucose_followup", # Shared for glucose monitoring + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - prediabetes", + }, + ), + ] + + +def init_glucose_diabetes(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood glucose diabetes range e126 mg/dL fasting.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_glucose_monitor", + arguments={ + "has_monitor": True, + "glucose_reading": 145, + "measurement_time": "fasting", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_glucose_followup", # Shared for glucose monitoring + "patient_id": "patient_001", + "reason": "Pending glucose monitoring follow-up - diabetes", + }, + ), + ] + + +def init_glucose_hypoglycemia(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Blood glucose hypoglycemia <70 mg/dL.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_glucose_monitor", + arguments={ + "has_monitor": True, + "glucose_reading": 62, + "measurement_time": "random", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Feeling shaky and dizzy", + "severity": "moderate", + "duration": "30 minutes", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_chronic_emergency_transfer", # For emergency transfers + "patient_id": "patient_001", + "reason": "Pending emergency transfer - hypoglycemia", + }, + ), + ] + + +def fix_glucose_monitoring(env: HealthcareEnvironment) -> list[ToolCall]: + """Schedule appointment for glucose monitoring.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_glucose", arguments={}), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "09:00", + "reason": "Blood glucose monitoring and diabetes management", + }, + ), + ] + + +def fix_glucose_hypoglycemia_emergency(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for hypoglycemia.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_glucose", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def init_spo2_normal(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Oxygen saturation normal 95-100%.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_pulse_oximeter", + arguments={"has_monitor": True, "spo2": 97, "heart_rate": 72}, + ) + ] + + +def init_spo2_mild_hypoxemia(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Oxygen saturation mild hypoxemia 90-94%.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_pulse_oximeter", + arguments={"has_monitor": True, "spo2": 92, "heart_rate": 78}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Mild shortness of breath", + "severity": "mild", + "duration": "few hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_spo2_followup", # Shared for oxygen monitoring + "patient_id": "patient_001", + "reason": "Pending oxygen saturation monitoring - mild hypoxemia", + }, + ), + ] + + +def init_spo2_moderate_hypoxemia(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Oxygen saturation moderate hypoxemia 85-89%.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_pulse_oximeter", + arguments={"has_monitor": True, "spo2": 87, "heart_rate": 88}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Moderate shortness of breath, feeling winded", + "severity": "moderate", + "duration": "several hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_spo2_followup", # Shared for oxygen monitoring + "patient_id": "patient_001", + "reason": "Pending oxygen saturation monitoring - moderate hypoxemia", + }, + ), + ] + + +def init_spo2_severe_hypoxemia(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Oxygen saturation severe hypoxemia <85%.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_pulse_oximeter", + arguments={"has_monitor": True, "spo2": 82, "heart_rate": 95}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Severe difficulty breathing, gasping for air", + "severity": "severe", + "duration": "1 hour", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_chronic_emergency_transfer", # For emergency transfers + "patient_id": "patient_001", + "reason": "Pending emergency transfer - severe hypoxemia", + }, + ), + ] + + +def fix_spo2_monitoring(env: HealthcareEnvironment) -> list[ToolCall]: + """Schedule appointment for oxygen monitoring.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_oxygen_saturation", arguments={}), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "11:00", + "reason": "Low oxygen saturation requiring urgent evaluation", + }, + ), + ] + + +def fix_spo2_emergency(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for severe hypoxemia.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_oxygen_saturation", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### Composition Functions + + +def consolidate_chronic_monitoring_appointments( + env: HealthcareEnvironment, fix_funcs: list +) -> list[ToolCall]: + """ + Consolidate multiple appointment bookings into a single comprehensive appointment. + + When multiple non-critical vitals need monitoring, medical practice dictates ONE + comprehensive appointment covering all conditions, not separate appointments. + + This function: + 1. Identifies all unique appointment booking actions from fix_funcs + 2. If multiple appointments would be booked, consolidates them into ONE + 3. Combines the appointment reasons into a comprehensive reason + 4. Keeps all other actions (identity verification, vital measurements, etc.) + """ + all_tool_calls = [] + for func in fix_funcs: + if func is not None: + all_tool_calls.extend(func(env)) + + deduplicated_calls = [] + seen_identity_verification = False + seen_check_slots = False + seen_check_calendar = False + appointment_calls = [] + + for tc in all_tool_calls: + if tc.name == "get_patient_details" and tc.requestor == "assistant": + if not seen_identity_verification: + deduplicated_calls.append(tc) + seen_identity_verification = True + elif tc.name == "check_available_time_slots" and tc.requestor == "assistant": + if not seen_check_slots: + deduplicated_calls.append(tc) + seen_check_slots = True + elif tc.name == "check_calendar" and tc.requestor == "user": + if not seen_check_calendar: + deduplicated_calls.append(tc) + seen_check_calendar = True + elif tc.name == "book_appointment": + appointment_calls.append(tc) + else: + deduplicated_calls.append(tc) + + if len(appointment_calls) > 1: + reasons = [call.arguments.get("reason", "") for call in appointment_calls] + consolidated_reason = "Review home monitoring readings - " + ", ".join( + set( + [ + r.replace( + "Follow-up for elevated blood pressure monitoring", + "blood pressure", + ) + .replace( + "Blood glucose monitoring and diabetes management", + "glucose/diabetes", + ) + .replace( + "Low oxygen saturation requiring urgent evaluation", + "oxygen saturation", + ) + for r in reasons + if r + ] + ) + ) + + consolidated_appointment = ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": appointment_calls[0].arguments["patient_id"], + "doctor_id": appointment_calls[0].arguments["doctor_id"], + "appointment_type": "follow_up", + "date": appointment_calls[0].arguments["date"], + "time": appointment_calls[0].arguments["time"], + "reason": consolidated_reason, + }, + ) + deduplicated_calls.append(consolidated_appointment) + elif len(appointment_calls) == 1: + deduplicated_calls.append(appointment_calls[0]) + + return deduplicated_calls + + +### Base Tasks + +bp_normal_task = BaseTask( + name="BP_normal", + description="Blood pressure normal <120/80", + init_funcs=[init_bp_normal], + fix_funcs=[fix_bp_normal_routine], +) + +bp_elevated_task = BaseTask( + name="elevated", + description="Blood pressure elevated 120-129/<80", + init_funcs=[init_bp_elevated], + fix_funcs=[fix_bp_elevated_monitor], +) + +bp_stage1_task = BaseTask( + name="stage1", + description="Stage 1 Hypertension 130-139/80-89", + init_funcs=[init_bp_stage1], + fix_funcs=[fix_bp_elevated_monitor], +) + +bp_stage2_task = BaseTask( + name="stage2", + description="Stage 2 Hypertension e140/90", + init_funcs=[init_bp_stage2], + fix_funcs=[fix_bp_elevated_monitor], +) + +bp_hypertensive_crisis_task = BaseTask( + name="hypertensive_crisis", + description="Hypertensive Crisis e180/120", + init_funcs=[init_bp_hypertensive_crisis], + fix_funcs=[fix_bp_crisis_emergency], +) + +glucose_normal_task = BaseTask( + name="normal", + description="Blood glucose normal 70-100 mg/dL", + init_funcs=[init_glucose_normal], + fix_funcs=[fix_glucose_normal_routine], +) + +glucose_prediabetes_task = BaseTask( + name="prediabetes", + description="Prediabetes range 100-125 mg/dL", + init_funcs=[init_glucose_prediabetes], + fix_funcs=[fix_glucose_monitoring], +) + +glucose_diabetes_task = BaseTask( + name="diabetes", + description="Diabetes range e126 mg/dL fasting", + init_funcs=[init_glucose_diabetes], + fix_funcs=[fix_glucose_monitoring], +) + +glucose_hypoglycemia_task = BaseTask( + name="hypoglycemia", + description="Hypoglycemia <70 mg/dL", + init_funcs=[init_glucose_hypoglycemia], + fix_funcs=[fix_glucose_hypoglycemia_emergency], +) + +spo2_normal_task = BaseTask( + name="normal", + description="Oxygen saturation normal 95-100%", + init_funcs=[init_spo2_normal], + fix_funcs=[fix_spo2_normal_routine], +) + +spo2_mild_hypoxemia_task = BaseTask( + name="mild_hypoxemia", + description="Mild hypoxemia 90-94%", + init_funcs=[init_spo2_mild_hypoxemia], + fix_funcs=[fix_spo2_monitoring], +) + +spo2_moderate_hypoxemia_task = BaseTask( + name="moderate_hypoxemia", + description="Moderate hypoxemia 85-89%", + init_funcs=[init_spo2_moderate_hypoxemia], + fix_funcs=[fix_spo2_emergency], +) + +spo2_severe_hypoxemia_task = BaseTask( + name="severe_hypoxemia", + description="Severe hypoxemia <85%", + init_funcs=[init_spo2_severe_hypoxemia], + fix_funcs=[fix_spo2_emergency], +) + + +### SelectionSets + +blood_pressure_issues = SelectionSet( + tasks=[ + bp_elevated_task, + bp_stage1_task, + bp_stage2_task, + ] +) + +blood_glucose_issues = SelectionSet( + tasks=[ + glucose_prediabetes_task, + glucose_diabetes_task, + ] +) + +oxygen_saturation_issues = SelectionSet( + tasks=[ + spo2_mild_hypoxemia_task, + ] +) + +chronic_monitoring_selection_sets = [ + blood_pressure_issues, + blood_glucose_issues, + oxygen_saturation_issues, +] + + +### Custom Composition for Chronic Monitoring + + +def consolidate_init_appointment_markers( + init_funcs: list, +) -> list: + """ + Consolidate multiple appointment markers in init functions. + + When multiple appointment markers would be created, consolidate them into one + generic "pending_chronic_monitoring" marker. + """ + consolidated_init_funcs = [] + + for func in init_funcs: + consolidated_init_funcs.append(func) + + # Create a wrapper that will consolidate appointment markers + def create_consolidated_init(): + def consolidated_init(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + all_calls = [] + appointment_markers = [] + + for func in init_funcs: + calls = func(env) + for call in calls: + if ( + isinstance(call, EnvFunctionCall) + and call.func_name == "create_appointment_marker" + ): + appointment_markers.append(call) + else: + all_calls.append(call) + + if len(appointment_markers) > 1: + all_calls.append( + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_chronic_monitoring", + "patient_id": "patient_001", + "reason": "Pending chronic condition monitoring follow-up", + }, + ) + ) + elif len(appointment_markers) == 1: + all_calls.append(appointment_markers[0]) + + return all_calls + + return consolidated_init + + return [create_consolidated_init()] + + +def compose_chronic_monitoring_tasks() -> list[ComposedTask]: + """ + Compose chronic monitoring tasks with custom logic to consolidate appointments. + + When multiple non-critical vitals need monitoring, creates tasks that expect ONE + comprehensive appointment instead of multiple separate appointments. + """ + from itertools import product + + selection_sets = chronic_monitoring_selection_sets + product_tasks = list( + product(*[selection_set.tasks + [None] for selection_set in selection_sets]) + ) + composed_tasks = [] + + for tasks in product_tasks: + tasks = sorted([t for t in tasks if t is not None], key=lambda x: x.name) + if len(tasks) == 0: + continue + + init_funcs_raw = [f for t in tasks for f in t.init_funcs] + has_emergency = any(None in t.fix_funcs for t in tasks) + non_none_fix_funcs = [f for t in tasks for f in t.fix_funcs if f is not None] + + if has_emergency: + fix_funcs = [None] + init_funcs = init_funcs_raw + elif len(non_none_fix_funcs) > 1: + init_funcs = consolidate_init_appointment_markers(init_funcs_raw) + + def create_consolidated_fix_func(funcs_to_consolidate): + def consolidated_fix(env: HealthcareEnvironment) -> list[ToolCall]: + return consolidate_chronic_monitoring_appointments( + env, funcs_to_consolidate + ) + + return consolidated_fix + + fix_funcs = [create_consolidated_fix_func(non_none_fix_funcs)] + elif len(non_none_fix_funcs) == 1: + fix_funcs = non_none_fix_funcs + init_funcs = init_funcs_raw + else: + fix_funcs = [] + init_funcs = init_funcs_raw + + extra_env_assertions = [f for t in tasks for f in t.extra_env_assertions] + + composed_task = ComposedTask( + name="|".join([t.name for t in tasks]), + description=", ".join([t.description for t in tasks]), + composed_from=tasks, + init_funcs=init_funcs, + fix_funcs=fix_funcs, + extra_env_assertions=extra_env_assertions, + ) + composed_tasks.append(composed_task) + + return composed_tasks diff --git a/src/tau2/domains/healthcare/tasks/const.py b/src/tau2/domains/healthcare/tasks/const.py new file mode 100644 index 00000000..fe6d6fe4 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/const.py @@ -0,0 +1,26 @@ +TOOL_CALL_INFO_CHECK = "If the tool call does not return updated information, you might need to perform another tool call to get the updated details." + +TOOL_CALL_GROUNDING = """ +Whenever the agent asks you about your medical information or documents, always ground your responses on the results of tool calls. +For example: If the agent asks about your insurance, always ground your response on the results of the `check_insurance_card` tool call. If the agent asks about symptoms, always ground your response on the results of the `check_symptoms` tool call. +Never make up information, always ground your responses on the results of tool calls. +If you are unsure about whether an action is necessary, always ask the agent for clarification. +""" + +PERSONA_1 = """ +As a 42-year-old project manager, you're comfortable with technology and medical systems. You've been managing your own healthcare for years and understand basic medical terminology. + +Your health literacy is above average - you can navigate patient portals, understand insurance terminology, and follow medical instructions without much difficulty. You're organized and keep track of your medications and appointments. + +In interactions, you're efficient and to-the-point. You provide requested information clearly and ask focused questions when you need clarification. You appreciate when healthcare staff respect your time and give you actionable next steps. +""" + +PERSONA_2 = """ +At 68 years old, you're a retired teacher managing several chronic conditions. Medical systems and healthcare terminology can be overwhelming, and you often need extra help navigating the system. + +Your health literacy is limited - terms like "copay," "prior authorization," and "formulary" confuse you. You have trouble remembering medication names and often refer to them by what they're for ("my blood pressure pill"). You prefer when someone walks you through each step slowly. + +When dealing with healthcare, you get anxious easily. You worry about making mistakes with your medications or missing important appointments. You need frequent reassurance and may ask the same question multiple times to make sure you understood correctly. +""" + +PERSONAS = {"None": None, "Easy": PERSONA_1, "Hard": PERSONA_2} diff --git a/src/tau2/domains/healthcare/tasks/create_tasks.py b/src/tau2/domains/healthcare/tasks/create_tasks.py new file mode 100644 index 00000000..1973a472 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/create_tasks.py @@ -0,0 +1,346 @@ +import json +import random +from argparse import ArgumentParser +from collections import defaultdict + +from tau2.data_model.tasks import Task +from tau2.domains.healthcare.tasks.prescription_issues import ( + prescription_refill_selection_sets, + is_fixed_prescription_refill, + get_env_assertions_prescription_refill, +) +from tau2.domains.healthcare.tasks.appointment_issues import ( + appointment_scheduling_selection_sets, +) +from tau2.domains.healthcare.tasks.urgent_triage_issues import ( + urgent_triage_selection_sets, +) +from tau2.domains.healthcare.tasks.chronic_monitoring_issues import ( + chronic_monitoring_selection_sets, + compose_chronic_monitoring_tasks, +) +from tau2.domains.healthcare.tasks.telehealth_issues import ( + telehealth_setup_selection_sets, +) +from tau2.domains.healthcare.tasks.test_results_issues import ( + test_results_selection_sets, +) +from tau2.domains.healthcare.tasks.patient_mistake_issues import ( + patient_mistake_selection_sets, +) +from tau2.domains.healthcare.tasks.critical_triage_issues import ( + critical_triage_selection_sets, + is_fixed_critical_triage, + get_env_assertions_critical_triage, +) +from tau2.domains.healthcare.tasks.manager import TaskManager +from tau2.domains.healthcare.tasks.const import ( + TOOL_CALL_GROUNDING, + TOOL_CALL_INFO_CHECK, +) +from tau2.domains.healthcare.tasks.utils import get_persona_from_task_id +from tau2.domains.healthcare.tasks.evaluation_functions import ( + is_fixed_appointment_scheduling, + get_env_assertions_appointment_scheduling, + is_fixed_urgent_triage, + get_env_assertions_urgent_triage, + is_fixed_chronic_monitoring, + get_env_assertions_chronic_monitoring, + is_fixed_telehealth_setup, + get_env_assertions_telehealth_setup, + is_fixed_test_results_access, + get_env_assertions_test_results_access, +) +from tau2.domains.healthcare.tasks.patient_mistake_issues import ( + is_fixed_patient_mistake, + get_env_assertions_patient_mistake, +) +from tau2.utils import DATA_DIR +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall + + +def get_env_assertions(expected_success: bool) -> list[EnvAssertion]: + """Placeholder for environment assertions.""" + return [] + + +def set_surrounding(env) -> list[EnvFunctionCall]: + """Set the patient info for task initialization.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_user_info", + arguments={ + "name": "Sarah Johnson", + "patient_id": "patient_001", + "date_of_birth": "1985-03-15", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_user_location", + arguments={"location": "home"}, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": False, + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +def is_fixed(env) -> bool: + """Placeholder: currently always returns True.""" + return True + + +prescription_refill_task_manager = TaskManager( + name="prescription_refill", + purpose="Test prescription refill request handling.", + task_instructions=f"Follow the agent's instructions. If they ask you to check medication bottles, use the check_medication_bottle tool. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You need to refill your prescription medication.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) is calling to request a prescription refill. Help them check prescription status and process refill if available, or guide them to contact their doctor if no refills remain.", + selection_sets=prescription_refill_selection_sets, + get_env_assertions=get_env_assertions_prescription_refill, + set_surrounding=set_surrounding, + is_fixed=is_fixed_prescription_refill, + domain="healthcare", +) + +appointment_scheduling_task_manager = TaskManager( + name="appointment_scheduling", + purpose="Test appointment scheduling with various constraints.", + task_instructions=f"Follow the agent's guidance. Check your calendar and insurance when asked. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You want to schedule a medical appointment.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) wants to schedule an appointment. Verify insurance, check doctor availability, and book appointment after getting patient confirmation.", + selection_sets=appointment_scheduling_selection_sets, + get_env_assertions=get_env_assertions_appointment_scheduling, + set_surrounding=set_surrounding, + is_fixed=is_fixed_appointment_scheduling, + domain="healthcare", +) + +urgent_triage_task_manager = TaskManager( + name="urgent_triage", + purpose="Test urgent care triage with symptom assessment.", + task_instructions=f"Describe your symptoms when asked. Use check_symptoms and take_temperature tools as directed. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You are not feeling well and need medical attention.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) is experiencing symptoms. Assess urgency based on fever level, pain severity, and breathing difficulty. Book urgent appointment or transfer to nurse as appropriate.", + selection_sets=urgent_triage_selection_sets, + get_env_assertions=get_env_assertions_urgent_triage, + set_surrounding=set_surrounding, + is_fixed=is_fixed_urgent_triage, + domain="healthcare", +) + +chronic_monitoring_task_manager = TaskManager( + name="chronic_monitoring", + purpose="Test chronic condition monitoring with home measurements.", + task_instructions=f"Share your home monitoring readings when asked. Use measure_blood_pressure, measure_blood_glucose, and measure_oxygen_saturation tools. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You want to discuss your home health monitoring readings.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}. You manage chronic health conditions.", + ticket="Patient {name} (DOB: {date_of_birth}) is calling about chronic condition monitoring. Review their blood pressure, glucose, and oxygen saturation readings. Schedule follow-up or transfer to nurse for concerning values.", + selection_sets=chronic_monitoring_selection_sets, + get_env_assertions=get_env_assertions_chronic_monitoring, + set_surrounding=set_surrounding, + is_fixed=is_fixed_chronic_monitoring, + domain="healthcare", +) + +telehealth_setup_task_manager = TaskManager( + name="telehealth_setup", + purpose="Test telehealth setup with consent and contact management.", + task_instructions=f"Provide consent when requested. Update emergency contact and acknowledge instructions using the appropriate tools. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You want to set up a telehealth appointment.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) wants to set up telehealth. Obtain necessary consent, verify emergency contact is current, and ensure patient acknowledges any medical instructions.", + selection_sets=telehealth_setup_selection_sets, + get_env_assertions=get_env_assertions_telehealth_setup, + set_surrounding=set_surrounding, + is_fixed=is_fixed_telehealth_setup, + domain="healthcare", +) + +test_results_task_manager = TaskManager( + name="test_results_access", + purpose="Test accessing and reviewing medical test results.", + task_instructions=f"Ask about your test results. Use open_patient_portal to view results when available. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You want to check your medical test results.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) is calling about test results. Check result status, direct to patient portal for ready results, or schedule follow-up for reviewed results. Transfer to nurse for critical findings.", + selection_sets=test_results_selection_sets, + get_env_assertions=get_env_assertions_test_results_access, + set_surrounding=set_surrounding, + is_fixed=is_fixed_test_results_access, + domain="healthcare", +) + +patient_mistake_task_manager = TaskManager( + name="patient_mistake", + purpose="Test handling of patient misinformation and verification workflows.", + task_instructions=f"Report your symptoms or medication concerns as you understand them. The agent may ask you to verify information using tools. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You have concerns about your symptoms or medication.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}.", + ticket="Patient {name} (DOB: {date_of_birth}) is calling with health concerns. Verify patient-reported information through objective tools (vitals, medication bottles). Use clinical reasoning to identify discrepancies between patient claims and objective data. CRITICAL SAFETY: Life-threatening symptoms (chest pain, difficulty breathing, severe pain) ALWAYS require immediate transfer to clinical staff, regardless of vital sign readings.", + selection_sets=patient_mistake_selection_sets, + get_env_assertions=get_env_assertions_patient_mistake, + set_surrounding=set_surrounding, + is_fixed=is_fixed_patient_mistake, + domain="healthcare", +) + +critical_triage_task_manager = TaskManager( + name="critical_triage", + purpose="Test recognition of critical vital signs requiring immediate escalation.", + task_instructions=f"Report your symptoms as described. You may underestimate the severity. Use home monitoring tools when asked. {TOOL_CALL_INFO_CHECK} {TOOL_CALL_GROUNDING}", + reason_for_call="You're not feeling well and want to check if you need an appointment.", + known_info="You are {name}, born on {date_of_birth}, currently at {location}. You have home health monitoring devices.", + ticket="Patient {name} (DOB: {date_of_birth}) is calling about symptoms. Patient may underestimate severity. CRITICAL: Measure vital signs objectively. Critical values (BP ≥180/120, glucose <70, O2 <85%) require IMMEDIATE transfer to clinical staff, even if patient reports mild symptoms.", + selection_sets=critical_triage_selection_sets, + get_env_assertions=get_env_assertions_critical_triage, + set_surrounding=set_surrounding, + is_fixed=is_fixed_critical_triage, + domain="healthcare", +) + + +def create_tasks(save_tasks: bool = True, max_count_per_bin: int = 3) -> list[Task]: + tasks: list[Task] = [] + + prescription_tasks = prescription_refill_task_manager.create_tasks(save_tasks=False) + print(f"Number of prescription refill tasks: {len(prescription_tasks)}") + tasks.extend(prescription_tasks) + + appointment_tasks = appointment_scheduling_task_manager.create_tasks( + save_tasks=False + ) + print(f"Number of appointment scheduling tasks: {len(appointment_tasks)}") + tasks.extend(appointment_tasks) + + urgent_tasks = urgent_triage_task_manager.create_tasks(save_tasks=False) + print(f"Number of urgent triage tasks: {len(urgent_tasks)}") + tasks.extend(urgent_tasks) + + # Use custom composition for chronic monitoring to consolidate appointments + chronic_composed_tasks = compose_chronic_monitoring_tasks() + chronic_tasks = chronic_monitoring_task_manager.create_tasks( + save_tasks=False, custom_composed_tasks=chronic_composed_tasks + ) + print(f"Number of chronic monitoring tasks: {len(chronic_tasks)}") + tasks.extend(chronic_tasks) + + telehealth_tasks = telehealth_setup_task_manager.create_tasks(save_tasks=False) + print(f"Number of telehealth setup tasks: {len(telehealth_tasks)}") + tasks.extend(telehealth_tasks) + + test_results_tasks = test_results_task_manager.create_tasks(save_tasks=False) + print(f"Number of test results access tasks: {len(test_results_tasks)}") + tasks.extend(test_results_tasks) + + patient_mistake_tasks = patient_mistake_task_manager.create_tasks(save_tasks=False) + print(f"Number of patient mistake tasks: {len(patient_mistake_tasks)}") + tasks.extend(patient_mistake_tasks) + + critical_triage_tasks = critical_triage_task_manager.create_tasks(save_tasks=False) + print(f"Number of critical triage tasks: {len(critical_triage_tasks)}") + tasks.extend(critical_triage_tasks) + + print(f"Number of tasks: {len(tasks)}") + + file = DATA_DIR / "tau2" / "domains" / "healthcare" / f"tasks_full.json" + if save_tasks: + with open(file, "w") as f: + json.dump([t.model_dump(exclude_unset=True) for t in tasks], f, indent=2) + + tasks_with_attrs = [] + for intent_tasks, intent in [ + (prescription_tasks, "prescription_refill"), + (appointment_tasks, "appointment_scheduling"), + (urgent_tasks, "urgent_triage"), + (chronic_tasks, "chronic_monitoring"), + (telehealth_tasks, "telehealth_setup"), + (test_results_tasks, "test_results_access"), + (patient_mistake_tasks, "patient_mistake"), + (critical_triage_tasks, "critical_triage"), + ]: + for task in intent_tasks: + num_subtasks = len(task.id.split("|")) + tasks_with_attrs.append( + { + "task": task, + "intent": intent, + "num_subtasks": num_subtasks, + "persona": get_persona_from_task_id(task.id), + } + ) + + file_small = DATA_DIR / "tau2" / "domains" / "healthcare" / f"tasks_small.json" + small_tasks = [t["task"] for t in tasks_with_attrs if t["num_subtasks"] == 1] + print(f"Number of tasks in small set: {len(small_tasks)}") + if save_tasks: + with open(file_small, "w") as f: + json.dump( + [t.model_dump(exclude_unset=True) for t in small_tasks], f, indent=2 + ) + + file_sampled = DATA_DIR / "tau2" / "domains" / "healthcare" / f"tasks.json" + + tasks_by_bins = defaultdict(list) + for task in tasks_with_attrs: + # Keep tasks with 2+ subtasks, except critical_triage (important despite 1 subtask) + if task["num_subtasks"] < 2 and task["intent"] != "critical_triage": + continue + tasks_by_bins[(task["intent"], task["num_subtasks"], task["persona"])].append( + task["task"] + ) + + sampled_tasks = [] + for (intent, num_subtasks, persona), tasks_in_bin in tasks_by_bins.items(): + num_sampled = min(max_count_per_bin, len(tasks_in_bin)) + sampled_tasks.extend(random.sample(tasks_in_bin, num_sampled)) + print( + f"Sampled {num_sampled} tasks for {intent} with {num_subtasks} subtasks and persona {persona}..." + ) + + action_counts = [ + len(task.evaluation_criteria.actions or []) for task in sampled_tasks + ] + simple = sum(1 for c in action_counts if c <= 2) + medium = sum(1 for c in action_counts if 3 <= c <= 4) + hard = sum(1 for c in action_counts if c >= 5) + + print(f"\nFinal task distribution:") + print(f" Total sampled: {len(sampled_tasks)}") + print(f" Natural complexity distribution (0-2 / 3-4 / 5+):") + print(f" Simple (0-2): {simple} ({simple / len(sampled_tasks) * 100:.1f}%)") + print(f" Medium (3-4): {medium} ({medium / len(sampled_tasks) * 100:.1f}%)") + print(f" Hard (5+): {hard} ({hard / len(sampled_tasks) * 100:.1f}%)") + if save_tasks: + with open(file_sampled, "w") as f: + json.dump( + [t.model_dump(exclude_unset=True) for t in sampled_tasks], f, indent=2 + ) + + return tasks + + +def main(): + parser = ArgumentParser() + parser.add_argument("-s", "--seed", type=int, default=42) + parser.add_argument("-m", "--max-count-per-bin", type=int, default=3) + args = parser.parse_args() + random.seed(args.seed) + create_tasks(max_count_per_bin=args.max_count_per_bin) + + +if __name__ == "__main__": + main() diff --git a/src/tau2/domains/healthcare/tasks/critical_triage_issues.py b/src/tau2/domains/healthcare/tasks/critical_triage_issues.py new file mode 100644 index 00000000..f4f06d3f --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/critical_triage_issues.py @@ -0,0 +1,251 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### Critical Blood Pressure + + +def init_hypertensive_crisis_standalone( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Patient reports mild headache, but BP is critically high (hypertensive crisis).""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_bp_monitor", + arguments={ + "has_monitor": True, + "systolic": 192, + "diastolic": 128, + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Mild headache, feeling a bit off", + "severity": "mild", + "duration": "2 hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_bp_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports headache, needs BP check", + }, + ), + ] + + +def fix_hypertensive_crisis_standalone(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient, measure BP, recognize crisis level, transfer immediately.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_pressure", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### Critical Hypoglycemia + + +def init_hypoglycemia_standalone(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Patient reports feeling dizzy/shaky, but glucose is dangerously low.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_glucose_monitor", + arguments={ + "has_monitor": True, + "glucose_reading": 52, # <70 is hypoglycemia, <54 is critical + "measurement_time": "fasting", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Feeling dizzy and shaky, a bit confused", + "severity": "moderate", + "duration": "30 minutes", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_glucose_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - diabetic patient with dizziness", + }, + ), + ] + + +def fix_hypoglycemia_standalone(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient, measure glucose, recognize critical hypoglycemia, transfer.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_blood_glucose", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### Severe Hypoxemia + + +def init_severe_hypoxemia_standalone( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Patient reports shortness of breath, oxygen saturation is dangerously low.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_pulse_oximeter", + arguments={ + "has_monitor": True, + "spo2": 82, # <90 is low, <85 is severe + "heart_rate": 92, # Normal range for context + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Short of breath when walking, tired", + "severity": "moderate", + "duration": "few days", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_oxygen_crisis_triage", + "patient_id": "patient_001", + "reason": "Pending triage - patient reports breathing difficulty", + }, + ), + ] + + +def fix_severe_hypoxemia_standalone(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient, measure O2, recognize severe hypoxemia, transfer.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="measure_oxygen_saturation", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### Base Tasks + +hypertensive_crisis_standalone_task = BaseTask( + name="hypertensive_crisis_standalone", + description="Patient reports mild headache; measured BP shows hypertensive crisis (192/128)", + init_funcs=[init_hypertensive_crisis_standalone], + fix_funcs=[fix_hypertensive_crisis_standalone], +) + +hypoglycemia_standalone_task = BaseTask( + name="hypoglycemia_standalone", + description="Patient reports dizziness; measured glucose shows critical hypoglycemia (52 mg/dL)", + init_funcs=[init_hypoglycemia_standalone], + fix_funcs=[fix_hypoglycemia_standalone], +) + +severe_hypoxemia_standalone_task = BaseTask( + name="severe_hypoxemia_standalone", + description="Patient reports shortness of breath; measured O2 shows severe hypoxemia (82%)", + init_funcs=[init_severe_hypoxemia_standalone], + fix_funcs=[fix_severe_hypoxemia_standalone], +) + +### Selection Set - These are standalone, not composed + +critical_triage_standalone_tasks = SelectionSet( + tasks=[ + hypertensive_crisis_standalone_task, + hypoglycemia_standalone_task, + severe_hypoxemia_standalone_task, + ] +) + +critical_triage_selection_sets = [ + critical_triage_standalone_tasks, +] + + +### Evaluation Functions + + +def is_fixed_critical_triage(env: HealthcareEnvironment) -> bool: + """ + Critical triage tasks check if the transfer was handled. + + These are expected failure scenarios - the task is only "fixed" if + the agent recognized the critical values and transferred to nurse. + + Since we can't directly check if transfer happened from the environment, + we check if the appointment marker was resolved (removed). + """ + markers = [ + appt + for appt in env.tools.db.appointments.values() + if appt.appointment_id.startswith("pending_") + ] + return len(markers) == 0 + + +def get_env_assertions_critical_triage( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """ + Get environment assertions for critical_triage tasks. + + These tasks test multi-step clinical decision-making: + 1. Verify patient identity + 2. Measure vital signs objectively + 3. Recognize critical values and transfer + + Unlike patient_mistake tasks (which are expected failures with just transfer), + these tasks expect the agent to complete ALL 3 actions successfully. + """ + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "transfer_to_nurse"}, + assert_value=True, + message="Agent must transfer to nurse for critical vital signs", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "book_appointment"}, + assert_value=True, + message="Agent incorrectly tried to book appointment for critical emergency", + ), + ] + + nl_assertions = [] + communicate_info = [] + + return (env_assertions, nl_assertions, communicate_info) diff --git a/src/tau2/domains/healthcare/tasks/evaluation_functions.py b/src/tau2/domains/healthcare/tasks/evaluation_functions.py new file mode 100644 index 00000000..5ef7fb0c --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/evaluation_functions.py @@ -0,0 +1,346 @@ +"""Evaluation functions for healthcare domain intents.""" + +from tau2.data_model.tasks import EnvAssertion +from tau2.domains.healthcare.environment import HealthcareEnvironment + + +# ============================================================================ +# APPOINTMENT_SCHEDULING evaluation functions +# ============================================================================ + + +def get_env_assertions_appointment_scheduling( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for appointment_scheduling tasks.""" + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_appointment_count_exceeds_baseline", + arguments={}, + assert_value=True, + message="No new appointment was created - appointment count did not exceed baseline", + ) + ] + + nl_assertions = [ + # "Agent verified patient identity before accessing appointment information", + # "Agent confirmed insurance coverage and copay amount with patient", + # "Agent checked doctor availability before suggesting appointment times", + # "Agent confirmed appointment details (date, time, doctor, reason) with patient before booking", + # "Agent provided clear appointment confirmation information" + ] + + return (env_assertions, nl_assertions, []) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "get_patient_details"}, + assert_value=True, + message="Agent transferred without verifying patient identity first", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "book_appointment"}, + assert_value=True, + message="Agent incorrectly attempted to book appointment when transfer was required", + ), + ] + + nl_assertions = [ + # "Agent explained why appointment cannot be scheduled", + # "Agent provided alternative options or next steps" + ] + + return (env_assertions, nl_assertions, []) + + +def is_fixed_appointment_scheduling(env: HealthcareEnvironment) -> bool: + """Check if appointment scheduling request is resolved.""" + assertions, _, _ = get_env_assertions_appointment_scheduling(expected_success=True) + success = True + for assertion in assertions: + success = success and env.run_env_assertion( + assertion, + raise_assertion_error=False, + ) + return success + + +# ============================================================================ +# URGENT_TRIAGE evaluation functions +# ============================================================================ + + +def get_env_assertions_urgent_triage( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for urgent_triage tasks.""" + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_appointment_count_exceeds_baseline", + arguments={}, + assert_value=True, + message="No urgent appointment was created - appointment count did not exceed baseline", + ) + ] + + nl_assertions = [ + # "Agent assessed all reported symptoms systematically", + # "Agent checked for red flag symptoms (high fever, severe pain, breathing difficulty)", + # "Agent asked patient to measure temperature if available", + # "Agent made appropriate triage decision based on symptom severity", + # "Agent clearly communicated next steps to patient" + ] + + return (env_assertions, nl_assertions, []) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "check_symptoms"}, + assert_value=True, + message="Agent transferred without assessing symptoms first", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "transfer_to_nurse"}, + assert_value=True, + message="Agent identified severe symptoms but failed to transfer to clinical team", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "book_appointment"}, + assert_value=True, + message="Agent incorrectly attempted to book appointment when transfer was required", + ), + ] + + nl_assertions = [ + # "Agent recognized severe/critical symptoms requiring immediate clinical attention", + # "Agent explained need for nurse/clinical review", + # "Agent transferred patient promptly" + ] + + return (env_assertions, nl_assertions, []) + + +def is_fixed_urgent_triage(env: HealthcareEnvironment) -> bool: + """Check if urgent triage request is resolved.""" + assertions, _, _ = get_env_assertions_urgent_triage(expected_success=True) + success = True + for assertion in assertions: + success = success and env.run_env_assertion( + assertion, + raise_assertion_error=False, + ) + return success + + +# ============================================================================ +# CHRONIC_MONITORING evaluation functions +# ============================================================================ + + +def get_env_assertions_chronic_monitoring( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for chronic_monitoring tasks.""" + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_appointment_count_exceeds_baseline", + arguments={}, + assert_value=True, + message="No monitoring appointment was created - appointment count did not exceed baseline", + ) + ] + + nl_assertions = [ + # "Agent requested all relevant home monitoring readings (BP, glucose, SpO2 as appropriate)", + # "Agent asked patient to take measurements if not recently done", + # "Agent assessed whether readings are within normal ranges for patient's conditions", + # "Agent made appropriate recommendation based on readings", + # "Agent provided clear guidance on when to seek further care" + ] + + return (env_assertions, nl_assertions, []) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "transfer_to_nurse"}, + assert_value=True, + message="Agent detected critical vitals but failed to transfer to nurse for emergency care", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "book_appointment"}, + assert_value=True, + message="Agent incorrectly attempted to book appointment when transfer was required", + ), + ] + + nl_assertions = [ + # "Agent identified concerning vital sign readings requiring clinical review", + # "Agent explained need for nurse evaluation", + # "Agent transferred patient for immediate clinical assessment" + ] + + return (env_assertions, nl_assertions, []) + + +def is_fixed_chronic_monitoring(env: HealthcareEnvironment) -> bool: + """Check if chronic monitoring request is resolved.""" + assertions, _, _ = get_env_assertions_chronic_monitoring(expected_success=True) + success = True + for assertion in assertions: + success = success and env.run_env_assertion( + assertion, + raise_assertion_error=False, + ) + return success + + +# ============================================================================ +# TELEHEALTH_SETUP evaluation functions +# ============================================================================ + + +def is_fixed_telehealth_setup(env: HealthcareEnvironment) -> bool: + """Check if telehealth setup request is resolved.""" + consents_provided = set(env.user_tools.device.consents_provided or []) + acknowledged_instructions = set( + env.user_tools.device.acknowledged_instructions or [] + ) + emergency_contact = env.user_tools.surroundings.emergency_contact + + if emergency_contact is not None: + contact_str = str(emergency_contact) + if ( + "Old Contact" in contact_str + or "disconnected" in contact_str + or "MISSING" in contact_str + ): + return False + + if len(consents_provided) == 0 and len(acknowledged_instructions) == 0: + return True + + required_consents = {"telehealth", "data_sharing"} + required_instructions = {"medication", "post_care", "pre_surgery"} + + if len(consents_provided) > 0: + if not required_consents.issubset(consents_provided): + return False + + if len(acknowledged_instructions) > 0: + if not required_instructions.issubset(acknowledged_instructions): + return False + + return True + + +def get_env_assertions_telehealth_setup( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for telehealth_setup tasks.""" + if expected_success: + env_assertions = [] + + nl_assertions = [ + # "Agent obtained all required telehealth consents from patient", + # "Agent verified emergency contact information is current", + # "Agent ensured patient acknowledged all medical instructions", + # "Agent confirmed patient has necessary technology for telehealth session", + # "Agent provided clear information about telehealth appointment process" + ] + + return (env_assertions, nl_assertions, []) + else: + nl_assertions = [ + # "Agent identified missing required consents or information", + # "Agent explained what is needed to complete telehealth setup" + ] + + return ([], nl_assertions, []) + + +# ============================================================================ +# TEST_RESULTS_ACCESS evaluation functions +# ============================================================================ + + +def get_env_assertions_test_results_access( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for test_results_access tasks.""" + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_appointment_count_exceeds_baseline", + arguments={}, + assert_value=True, + message="No follow-up appointment was created - appointment count did not exceed baseline", + ) + ] + + nl_assertions = [ + # "Agent verified patient identity before discussing test results", + # "Agent checked test result status (ready, pending, or under review)", + # "Agent directed patient to appropriate resource (portal for ready results, follow-up for abnormalities)", + # "Agent explained test result findings in patient-friendly language if available", + # "Agent scheduled follow-up if abnormalities were detected" + ] + + return (env_assertions, nl_assertions, []) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "check_test_results"}, + assert_value=True, + message="Agent transferred without checking test results first", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "book_appointment"}, + assert_value=True, + message="Agent incorrectly attempted to book appointment when transfer was required", + ), + ] + + nl_assertions = [ + # "Agent identified critical/urgent test results requiring clinical review", + # "Agent did not release critical results directly to patient without clinical review", + # "Agent transferred patient to nurse for proper clinical evaluation" + ] + + return (env_assertions, nl_assertions, []) + + +def is_fixed_test_results_access(env: HealthcareEnvironment) -> bool: + """Check if test results access request is resolved.""" + assertions, _, _ = get_env_assertions_test_results_access(expected_success=True) + success = True + for assertion in assertions: + success = success and env.run_env_assertion( + assertion, + raise_assertion_error=False, + ) + return success diff --git a/src/tau2/domains/healthcare/tasks/manager.py b/src/tau2/domains/healthcare/tasks/manager.py new file mode 100644 index 00000000..5713d29d --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/manager.py @@ -0,0 +1,338 @@ +import json +import textwrap +from copy import deepcopy +from typing import Callable, Optional, Protocol, cast + +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall, Task +from tau2.domains.healthcare.environment import HealthcareEnvironment, get_environment +from tau2.utils import DATA_DIR + +from .const import PERSONAS +from .utils import BaseTask, ComposedTask, SelectionSet, compose_tasks + + +class GetEnvAssertionsCallable(Protocol): + """Protocol for get_env_assertions function. + + Returns tuple of (env_assertions, nl_assertions, communicate_info). + """ + + def __call__( + self, expected_success: bool + ) -> tuple[list[EnvAssertion], list[str], list[str]]: ... + + +def prepare_base_task(base_task: dict, env: HealthcareEnvironment) -> dict: + """Prepare task with patient-specific information.""" + base_task = deepcopy(base_task) + patient_name = env.user_tools.surroundings.full_name + date_of_birth = env.user_tools.surroundings.date_of_birth + location = env.user_tools.surroundings.location + + user_info = { + "name": patient_name, + "date_of_birth": date_of_birth, + "location": location, + } + + known_info_template = base_task["user_scenario"]["instructions"]["known_info"] + known_info = known_info_template.format(**user_info) + base_task["user_scenario"]["instructions"]["known_info"] = known_info + + ticket_template = base_task["ticket"] + ticket = ticket_template.format(**user_info) + base_task["ticket"] = ticket + + return base_task + + +class TaskManager: + def __init__( + self, + name: str, + purpose: str, + task_instructions: str, + reason_for_call: str, + known_info: str, + ticket: str, + selection_sets: list[SelectionSet], + get_env_assertions: GetEnvAssertionsCallable, + set_surrounding: Callable[[HealthcareEnvironment], list[EnvFunctionCall]], + is_fixed: Callable[[HealthcareEnvironment], bool], + task_validator: Optional[Callable[[list[Optional[BaseTask]]], bool]] = None, + domain: str = "healthcare", + ): + self.domain = domain + self.name = name + self.base_task_template = { + "id": f"[{name}]", + "description": { + "purpose": purpose, + }, + "user_scenario": { + "instructions": { + "task_instructions": task_instructions, + "domain": domain, + "reason_for_call": reason_for_call, + "known_info": known_info, + }, + "persona": None, + }, + "ticket": ticket, + "initial_state": {}, + "evaluation_criteria": {"env_assertions": None}, + } + + self.selection_sets = selection_sets + self.get_env_assertions = get_env_assertions + self.set_surrounding = set_surrounding + self.is_fixed = is_fixed + self.task_validator = task_validator + + def create_task(self, composed_task: ComposedTask, persona: str = "None") -> Task: + env = get_environment() + + init_actions = self.set_surrounding(env) + env.run_env_function_calls(init_actions) + for func in composed_task.init_funcs: + func_calls = func(env) + env.run_env_function_calls(func_calls) + init_actions.extend( + [fc for fc in func_calls if not isinstance(fc, EnvAssertion)] + ) + + fix_tool_calls: list[ToolCall] = [] + expected_failure = False + for func in composed_task.fix_funcs: + if func is None: + expected_failure = True + break + tool_calls = func(env) + fix_tool_calls.extend(tool_calls) + # Check if any fix function contains a transfer action + if any( + tc.name in {"transfer_to_nurse", "transfer_to_human_agent"} + for tc in tool_calls + ): + expected_failure = True + break + + deduplicated_tool_calls: list[ToolCall] = [] + seen_identity_verification = False + + for tc in fix_tool_calls: + if tc.name == "get_patient_details" and tc.requestor == "assistant": + if seen_identity_verification: + continue + else: + seen_identity_verification = True + + deduplicated_tool_calls.append(tc) + + fix_tool_calls = deduplicated_tool_calls + + if expected_failure: + fix_actions = [ + { + "action_id": "transfer_to_nurse", + "name": "transfer_to_nurse", + "requestor": "assistant", + "arguments": {}, + } + ] + else: + fix_actions = [] + for i, tc in enumerate(fix_tool_calls): + action = { + "action_id": f"{tc.name}_{i}", + "name": tc.name, + "requestor": tc.requestor, + "arguments": tc.arguments, + } + compare_args = getattr(tc, "compare_args", None) + if compare_args is not None: + action["compare_args"] = compare_args + fix_actions.append(action) + + env_assertions, nl_assertions, communicate_info = self.get_env_assertions( + expected_success=not expected_failure + ) + if not expected_failure: + for func in composed_task.extra_env_assertions: + extra_assertions = func(env) + env_assertions.extend(extra_assertions) + + outcome_focused_intents = { + "appointment_scheduling", + "test_results_access", + "chronic_monitoring", + "telehealth_setup", + "urgent_triage", + } + + intent_name = self.name + + if expected_failure: + reward_eval_mode = ["ACTION"] + elif intent_name in outcome_focused_intents: + # For outcome-focused tasks, use BOTH ENV_ASSERTION and ACTION + # to enforce correct outcomes AND safe procedures + if len(fix_actions) > 0 and len(env_assertions) > 0: + reward_eval_mode = ["ENV_ASSERTION", "ACTION"] + elif len(env_assertions) > 0: + reward_eval_mode = ["ENV_ASSERTION"] + else: + reward_eval_mode = ["ACTION"] + elif len(fix_actions) > 0 and len(env_assertions) > 0: + reward_eval_mode = ["ACTION", "ENV_ASSERTION"] + elif len(env_assertions) > 0: + reward_eval_mode = ["ENV_ASSERTION"] + else: + reward_eval_mode = ["ACTION"] + + final_task = prepare_base_task(self.base_task_template, env) + final_task["initial_state"]["initialization_actions"] = init_actions + final_task["evaluation_criteria"]["actions"] = fix_actions + final_task["evaluation_criteria"]["env_assertions"] = env_assertions + final_task["evaluation_criteria"]["nl_assertions"] = ( + nl_assertions if nl_assertions else None + ) + final_task["evaluation_criteria"]["communicate_info"] = ( + communicate_info if communicate_info else None + ) + final_task["evaluation_criteria"]["reward_basis"] = reward_eval_mode + final_task["user_scenario"]["persona"] = PERSONAS[persona] + final_task["id"] += f"{composed_task.name}[PERSONA:{persona}]" + final_task["description"]["info"] = composed_task.description + task = Task(**final_task) + return task + + def create_tasks( + self, + save_tasks: bool = False, + custom_composed_tasks: Optional[list[ComposedTask]] = None, + ) -> list[Task]: + if custom_composed_tasks is not None: + composed_tasks = custom_composed_tasks + else: + composed_tasks = compose_tasks(self.selection_sets, self.task_validator) + composed_tasks = sorted(composed_tasks, key=lambda x: len(x.composed_from)) + print(f"Number of composed tasks: {len(composed_tasks)}") + persona_options = list(PERSONAS.keys()) + personas = [ + persona_options[i % len(persona_options)] + for i in range(len(composed_tasks)) + ] + tasks = [] + for i, composed_task in enumerate(composed_tasks): + print(f"Task {i + 1}") + print(composed_task.name) + task = self.create_task(composed_task, personas[i]) + print(task) + print("-" * 100) + self.verify_task(task) + print("-" * 100) + tasks.append(task) + if save_tasks: + file = ( + DATA_DIR / "tau2" / "domains" / self.domain / f"{self.name}_tasks.json" + ) + with open(file, "w") as f: + json.dump([t.model_dump() for t in tasks], f, indent=2) + return tasks + + def run_assertions( + self, + env: HealthcareEnvironment, + task: Task, + verbose: bool = False, + skip_behavioral: bool = False, + ): + if task.evaluation_criteria is None: + return True + assertions = task.evaluation_criteria.env_assertions or [] + if len(assertions) == 0: + return True + success = True + for i, assertion in enumerate(assertions): + # Skip behavioral assertions (tool call history checks) during task verification + # These require full conversation trajectories and will be verified during evaluation + if skip_behavioral and assertion.func_name in [ + "assert_tool_was_called", + "assert_tool_was_not_called", + ]: + if verbose: + print( + f"Skipping behavioral assertion {i + 1} of {len(assertions)} (will be verified during evaluation)" + ) + print(textwrap.indent(str(assertion), " ")) + continue + + if verbose: + print(f"Verifying env assertion {i + 1} of {len(assertions)}") + print(textwrap.indent(str(assertion), " ")) + assertion_success = env.run_env_assertion( + assertion, + raise_assertion_error=False, + ) + if verbose: + print("Success: ", assertion_success) + success = success and assertion_success + return success + + def _is_fixable(self, task: Task) -> bool: + transfer_action_names = {"transfer_to_human_agent", "transfer_to_nurse"} + if task.evaluation_criteria is None: + return True + action_names = {a.name for a in task.evaluation_criteria.actions or []} + if action_names & transfer_action_names: + return False + return True + + def verify_task(self, task: Task): + from tau2.registry import registry + + print("Verifying task: ", task.id) + + healthcare_env = cast( + HealthcareEnvironment, registry.get_env_constructor("healthcare")() + ) + assert self.is_fixed(healthcare_env), "Healthcare env starts in broken state" + + initialization_data = None + initialization_actions = None + if task.initial_state is not None: + initialization_data = task.initial_state.initialization_data + initialization_actions = task.initial_state.initialization_actions + + healthcare_env.set_state( + initialization_data=initialization_data, + initialization_actions=initialization_actions, + message_history=[], + ) + + fix_actions = [] + if task.evaluation_criteria is not None: + fix_actions = task.evaluation_criteria.actions or [] + + fixable = self._is_fixable(task) + for i, action in enumerate(fix_actions): + assert not self.is_fixed(healthcare_env), ( + f"Task {task.id} is already fixed after {i} actions. {task}" + ) + healthcare_env.make_tool_call( + tool_name=action.name, requestor=action.requestor, **action.arguments + ) + healthcare_env.sync_tools() + if fixable: + assert self.is_fixed(healthcare_env), ( + f"Task {task.id} is not fixed after all actions. {task}" + ) + else: + assert not self.is_fixed(healthcare_env), ( + f"Task {task.id} is fixed but should not be. {task}" + ) + assert self.run_assertions( + healthcare_env, task, verbose=True, skip_behavioral=True + ) diff --git a/src/tau2/domains/healthcare/tasks/patient_mistake_issues.py b/src/tau2/domains/healthcare/tasks/patient_mistake_issues.py new file mode 100644 index 00000000..5a2d2e1c --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/patient_mistake_issues.py @@ -0,0 +1,329 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + +### SelectionSet 11: symptom_misreporting_issues + + +def init_chest_pain_vs_panic_attack( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Patient reports chest pain. Agent must verify vitals and escalate to nurse.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 98.4}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Severe chest pain radiating to left arm, feels like heart attack, racing heart", + "severity": "severe", + "duration": "20 minutes", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending triage - chest pain requiring verification", + }, + ), + ] + + +def init_high_fever_vs_normal_temp(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Patient reports fever but temperature is normal.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 98.6}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Feeling feverish, very hot and sweaty", + "severity": "mild", + "duration": "4 hours", + }, + ), + ] + + +def fix_verify_chest_pain_mismatch(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient identity, check symptoms and vitals, transfer to nurse.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_symptoms", arguments={}), + ToolCall(requestor="user", name="take_temperature", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def fix_verify_fever_mismatch(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient identity, check symptoms and temperature.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_symptoms", arguments={}), + ToolCall(requestor="user", name="take_temperature", arguments={}), + ] + + +### SelectionSet 12: medication_confusion_issues + + +def init_wrong_medication_not_working( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Patient confuses medication names. Claims wrong medication isn't working for blood pressure.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "active"}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_003", + "medication_name": "Atorvastatin", + "dosage": "20mg at bedtime", + "refills_remaining": 3, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication confusion verification", + }, + ), + ] + + +def init_wrong_dosage_confusion(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Patient reports taking incorrect dosage.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "active"}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": "Lisinopril", + "dosage": "10mg once daily", + "refills_remaining": 2, + "prescribing_doctor": "Dr. Sarah Martinez", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_symptom_verification", + "patient_id": "patient_001", + "reason": "Pending medication dosage verification", + }, + ), + ] + + +def fix_verify_medication_confusion(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient identity, check medication bottles, look up prescriptions.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_medication_bottle", arguments={}), + ToolCall( + requestor="assistant", + name="get_prescription_details", + arguments={"prescription_id": "rx_001"}, + ), + ToolCall( + requestor="assistant", + name="get_prescription_details", + arguments={"prescription_id": "rx_003"}, + ), + ] + + +def fix_verify_dosage_confusion(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient identity, check medication bottle, verify prescription, transfer to nurse.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_medication_bottle", arguments={}), + ToolCall( + requestor="assistant", + name="get_prescription_details", + arguments={"prescription_id": "rx_001"}, + ), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### Base Tasks for SelectionSet 11: symptom_misreporting_issues + +chest_pain_vs_panic_attack_task = BaseTask( + name="chest_pain_vs_panic_attack", + description="Patient reports severe chest pain; agent must verify vitals and escalate appropriately", + init_funcs=[init_chest_pain_vs_panic_attack], + fix_funcs=[fix_verify_chest_pain_mismatch], +) + +high_fever_vs_normal_temp_task = BaseTask( + name="high_fever_vs_normal_temp", + description="Patient reports high fever, but temperature is normal", + init_funcs=[init_high_fever_vs_normal_temp], + fix_funcs=[], +) + +### Base Tasks for SelectionSet 12: medication_confusion_issues + +wrong_medication_not_working_task = BaseTask( + name="wrong_medication_not_working", + description="Patient claims wrong medication isn't working for their condition", + init_funcs=[init_wrong_medication_not_working], + fix_funcs=[None], +) + +wrong_dosage_confusion_task = BaseTask( + name="wrong_dosage_confusion", + description="Patient reports taking incorrect dosage", + init_funcs=[init_wrong_dosage_confusion], + fix_funcs=[None], +) + +### SelectionSets + +symptom_misreporting_issues = SelectionSet( + tasks=[ + chest_pain_vs_panic_attack_task, + high_fever_vs_normal_temp_task, + ] +) + +medication_confusion_issues = SelectionSet( + tasks=[ + wrong_medication_not_working_task, + wrong_dosage_confusion_task, + ] +) + +patient_mistake_selection_sets = [ + symptom_misreporting_issues, + medication_confusion_issues, +] + + +### Evaluation Functions + + +def is_fixed_patient_mistake(env: HealthcareEnvironment) -> bool: + """Check if patient mistake scenario is resolved.""" + symptom_marker = env.tools.db.appointments.get("pending_symptom_verification") + if symptom_marker is not None: + return False + + if len(env.user_tools.db.patient_device.medications_at_home) >= 2: + return False + + return True + + +def get_env_assertions_patient_mistake( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for patient_mistake tasks.""" + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "get_patient_details"}, + assert_value=True, + message="Agent must verify patient identity before addressing medical concerns", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "take_temperature"}, + assert_value=True, + message="Agent must verify patient's fever claim with objective temperature reading", + ), + ] + nl_assertions = [] + communicate_info = [] + return (env_assertions, nl_assertions, communicate_info) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "get_patient_details"}, + assert_value=True, + message="Agent must verify patient identity before addressing medical concerns", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "check_medication_bottle"}, + assert_value=True, + message="Agent must verify medication details before escalating confusion", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "transfer_to_nurse"}, + assert_value=True, + message="Agent must escalate patient mistake to nurse for clinical review", + ), + ] + nl_assertions = [] + return (env_assertions, nl_assertions, []) diff --git a/src/tau2/domains/healthcare/tasks/prescription_issues.py b/src/tau2/domains/healthcare/tasks/prescription_issues.py new file mode 100644 index 00000000..691f97d0 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/prescription_issues.py @@ -0,0 +1,429 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### SelectionSet 1: refills_status_issues + + +def init_no_refills_remaining( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Set prescription to have 0 refills remaining.""" + rx = env.tools.db.prescriptions["rx_001"] + doctor = env.tools.db.doctors[rx.doctor_id] + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_refills", + arguments={"prescription_id": "rx_001", "refills": 0}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": rx.medication_name, + "dosage": rx.dosage, + "refills_remaining": 0, + "prescribing_doctor": f"Dr. {doctor.name.first_name} {doctor.name.last_name}", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_refills_remaining", + arguments={"prescription_id": "rx_001", "expected_count": 0}, + assert_value=True, + ), + ] + + +def init_has_refills_available( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Set prescription to have refills available.""" + rx = env.tools.db.prescriptions["rx_001"] + doctor = env.tools.db.doctors[rx.doctor_id] + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_refills", + arguments={"prescription_id": "rx_001", "refills": 3}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "active"}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": rx.medication_name, + "dosage": rx.dosage, + "refills_remaining": 3, + "prescribing_doctor": f"Dr. {doctor.name.first_name} {doctor.name.last_name}", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_refills_remaining", + arguments={"prescription_id": "rx_001", "expected_count": 3}, + assert_value=True, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_status", + arguments={"prescription_id": "rx_001", "expected_status": "active"}, + assert_value=True, + ), + ] + + +def fix_has_refills_available(env: HealthcareEnvironment) -> list[ToolCall]: + """Process refill for prescription with refills available.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_medication_bottle", arguments={}), + ToolCall( + requestor="assistant", + name="get_prescription_details", + arguments={"prescription_id": "rx_001"}, + ), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={ + "patient_id": "patient_001", + "procedure_type": "prescription_refill", + }, + compare_args=["patient_id"], + ), + ToolCall( + requestor="assistant", + name="request_prescription_refill", + arguments={"patient_id": "patient_001", "prescription_id": "rx_001"}, + ), + ] + + +def fix_prescription_needs_renewal(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify prescription details then transfer to nurse for renewal.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_medication_bottle", arguments={}), + ToolCall( + requestor="assistant", + name="get_prescription_details", + arguments={"prescription_id": "rx_001"}, + ), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +### SelectionSet 2: prescription_status_issues + + +def init_prescription_active( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Set prescription status to active with refills available.""" + rx = env.tools.db.prescriptions["rx_001"] + doctor = env.tools.db.doctors[rx.doctor_id] + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "active"}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_refills", + arguments={"prescription_id": "rx_001", "refills": 3}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": rx.medication_name, + "dosage": rx.dosage, + "refills_remaining": 3, + "prescribing_doctor": f"Dr. {doctor.name.first_name} {doctor.name.last_name}", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_status", + arguments={"prescription_id": "rx_001", "expected_status": "active"}, + assert_value=True, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_refills_remaining", + arguments={"prescription_id": "rx_001", "expected_count": 3}, + assert_value=True, + ), + ] + + +def init_prescription_expired( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Set prescription status to expired.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "expired"}, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_status", + arguments={"prescription_id": "rx_001", "expected_status": "expired"}, + assert_value=True, + ), + ] + + +def init_prescription_discontinued( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Set prescription status to discontinued.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "discontinued"}, + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_status", + arguments={"prescription_id": "rx_001", "expected_status": "discontinued"}, + assert_value=True, + ), + ] + + +### SelectionSet 3: prescription_type_issues + + +def init_regular_medication(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Initialize with regular non-controlled medication with refills.""" + rx = env.tools.db.prescriptions["rx_001"] + doctor = env.tools.db.doctors[rx.doctor_id] + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_refills", + arguments={"prescription_id": "rx_001", "refills": 3}, + ), + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_status", + arguments={"prescription_id": "rx_001", "status": "active"}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_medication_at_home", + arguments={ + "prescription_number": "rx_001", + "medication_name": rx.medication_name, + "dosage": rx.dosage, + "refills_remaining": 3, + "prescribing_doctor": f"Dr. {doctor.name.first_name} {doctor.name.last_name}", + "pharmacy_name": "Community Pharmacy", + "pharmacy_phone": "(555) 123-4567", + }, + ), + ] + + +def init_controlled_substance(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Initialize with controlled substance.""" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="set_prescription_medication", + arguments={ + "prescription_id": "rx_001", + "medication_name": "Oxycodone", + "dosage": "5mg every 6 hours as needed", + }, + ) + ] + + +### Base Tasks + +no_refills_remaining_task = BaseTask( + name="no_refills_remaining", + description="Patient has 0 refills left on prescription", + init_funcs=[init_no_refills_remaining], + fix_funcs=[fix_prescription_needs_renewal], +) + +has_refills_available_task = BaseTask( + name="has_refills_available", + description="Patient has refills available", + init_funcs=[init_has_refills_available], + fix_funcs=[fix_has_refills_available], +) + +prescription_active_task = BaseTask( + name="prescription_active", + description="Prescription is active", + init_funcs=[init_prescription_active], + fix_funcs=[fix_has_refills_available], +) + +prescription_expired_task = BaseTask( + name="prescription_expired", + description="Prescription has expired", + init_funcs=[init_prescription_expired], + fix_funcs=[fix_prescription_needs_renewal], +) + +prescription_discontinued_task = BaseTask( + name="prescription_discontinued", + description="Prescription was discontinued", + init_funcs=[init_prescription_discontinued], + fix_funcs=[fix_prescription_needs_renewal], +) + +regular_medication_task = BaseTask( + name="regular_medication", + description="Regular non-controlled medication", + init_funcs=[init_regular_medication], + fix_funcs=[fix_has_refills_available], +) + +controlled_substance_task = BaseTask( + name="controlled_substance", + description="Controlled substance", + init_funcs=[init_controlled_substance], + fix_funcs=[fix_prescription_needs_renewal], +) + + +### SelectionSets + +refills_status_issues = SelectionSet( + tasks=[ + no_refills_remaining_task, + has_refills_available_task, + ] +) + +prescription_status_issues = SelectionSet( + tasks=[ + prescription_active_task, + prescription_expired_task, + prescription_discontinued_task, + ] +) + +prescription_type_issues = SelectionSet( + tasks=[ + regular_medication_task, + controlled_substance_task, + ] +) + +prescription_refill_selection_sets = [ + refills_status_issues, +] + + +### Evaluation Functions + + +def is_fixed_prescription_refill(env: HealthcareEnvironment) -> bool: + """Check if the prescription refill issue is resolved.""" + if "rx_001" not in env.tools.db.prescriptions: + return True + + rx = env.tools.db.prescriptions["rx_001"] + + is_default_state = ( + rx.refills_remaining == 2 + and rx.status == "active" + and rx.medication_name == "Lisinopril" + ) + + return is_default_state + + +def get_env_assertions_prescription_refill( + expected_success: bool, +) -> tuple[list[EnvAssertion], list[str], list[str]]: + """Get environment assertions for prescription_refill tasks. + + Args: + expected_success: Whether the task is expected to be successfully resolved + + Returns: + Tuple of (env_assertions, nl_assertions, communicate_info) + """ + if expected_success: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_status", + arguments={"prescription_id": "rx_001", "expected_status": "active"}, + assert_value=True, + message="Prescription status should be active for successful refill", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_prescription_refills_remaining", + arguments={"prescription_id": "rx_001", "expected_count": 2}, + assert_value=True, + message="Prescription should have 2 refills remaining after processing one refill", + ), + ] + + nl_assertions = [] + + return (env_assertions, nl_assertions, []) + else: + env_assertions = [ + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_called", + arguments={"tool_name": "get_prescription_details"}, + assert_value=True, + message="Agent transferred without checking prescription details first", + ), + EnvAssertion( + env_type="assistant", + func_name="assert_tool_was_not_called", + arguments={"tool_name": "request_prescription_refill"}, + assert_value=True, + message="Agent incorrectly attempted to refill prescription that requires nurse escalation", + ), + ] + + nl_assertions = [] + + return (env_assertions, nl_assertions, []) diff --git a/src/tau2/domains/healthcare/tasks/telehealth_issues.py b/src/tau2/domains/healthcare/tasks/telehealth_issues.py new file mode 100644 index 00000000..3766a523 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/telehealth_issues.py @@ -0,0 +1,346 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### Init Functions + + +def init_consent_not_required(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """No consent required for this interaction.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="provide_consent", + arguments={"consent_type": "telehealth"}, + ), + EnvFunctionCall( + env_type="user", + func_name="provide_consent", + arguments={"consent_type": "data_sharing"}, + ), + ] + + +def init_telehealth_consent_needed( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall | EnvAssertion]: + """Telehealth consent required but not yet provided.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="provide_consent", + arguments={"consent_type": "data_sharing"}, + ) + ] + + +def init_data_sharing_consent_needed( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Data sharing consent required for specialist referral.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="provide_consent", + arguments={"consent_type": "telehealth"}, + ) + ] + + +### Fix Functions + + +def fix_obtain_telehealth_consent(env: HealthcareEnvironment) -> list[ToolCall]: + """Obtain telehealth consent from patient.""" + return [ + ToolCall( + requestor="user", + name="provide_consent", + arguments={"consent_type": "telehealth"}, + ) + ] + + +def fix_obtain_data_sharing_consent(env: HealthcareEnvironment) -> list[ToolCall]: + """Obtain data sharing consent from patient.""" + return [ + ToolCall( + requestor="user", + name="provide_consent", + arguments={"consent_type": "data_sharing"}, + ) + ] + + +def init_emergency_contact_current(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Emergency contact is current and on file.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_emergency_contact", + arguments={ + "name": "Jane Smith", + "phone": "555-0102", + "relationship": "spouse", + }, + ) + ] + + +def init_emergency_contact_missing(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """No emergency contact on file.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_emergency_contact", + arguments={ + "name": "MISSING - No emergency contact on file", + "phone": "000-0000", + "relationship": "none", + }, + ) + ] + + +def init_emergency_contact_outdated( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Emergency contact information is outdated.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_emergency_contact", + arguments={ + "name": "Old Contact (disconnected)", + "phone": "555-9999", + "relationship": "friend", + }, + ) + ] + + +def fix_update_emergency_contact(env: HealthcareEnvironment) -> list[ToolCall]: + """Update emergency contact information.""" + return [ + ToolCall( + requestor="user", + name="update_emergency_contact", + arguments={ + "name": "Emergency Contact", + "phone": "555-0000", + "relationship": "family", + }, + compare_args=[], + ) + ] + + +def init_no_instructions_needed(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """No special instructions required.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "medication"}, + ), + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "post_care"}, + ), + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "pre_surgery"}, + ), + ] + + +def init_medication_instructions(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Medication instructions need acknowledgment.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "post_care"}, + ), + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "pre_surgery"}, + ), + ] + + +def init_post_care_instructions(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Post-care instructions need acknowledgment.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "medication"}, + ), + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "pre_surgery"}, + ), + ] + + +def init_pre_surgery_instructions(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Pre-surgery instructions need acknowledgment.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "medication"}, + ), + EnvFunctionCall( + env_type="user", + func_name="acknowledge_instructions", + arguments={"instruction_type": "post_care"}, + ), + ] + + +def fix_acknowledge_medication(env: HealthcareEnvironment) -> list[ToolCall]: + """Acknowledge medication instructions.""" + return [ + ToolCall( + requestor="user", + name="acknowledge_instructions", + arguments={"instruction_type": "medication"}, + compare_args=["instruction_type"], + ) + ] + + +def fix_acknowledge_post_care(env: HealthcareEnvironment) -> list[ToolCall]: + """Acknowledge post-care instructions.""" + return [ + ToolCall( + requestor="user", + name="acknowledge_instructions", + arguments={"instruction_type": "post_care"}, + compare_args=["instruction_type"], + ) + ] + + +def fix_acknowledge_pre_surgery(env: HealthcareEnvironment) -> list[ToolCall]: + """Acknowledge pre-surgery instructions.""" + return [ + ToolCall( + requestor="user", + name="acknowledge_instructions", + arguments={"instruction_type": "pre_surgery"}, + compare_args=["instruction_type"], + ) + ] + + +### Base Tasks + +consent_not_required_task = BaseTask( + name="not_required", + description="No consent required", + init_funcs=[], + fix_funcs=[], +) + +telehealth_consent_needed_task = BaseTask( + name="telehealth_consent_needed", + description="Telehealth consent required", + init_funcs=[init_telehealth_consent_needed], + fix_funcs=[fix_obtain_telehealth_consent], +) + +data_sharing_consent_needed_task = BaseTask( + name="data_sharing_consent_needed", + description="Data sharing consent required", + init_funcs=[init_data_sharing_consent_needed], + fix_funcs=[fix_obtain_data_sharing_consent], +) + +emergency_contact_current_task = BaseTask( + name="current", + description="Emergency contact current and on file", + init_funcs=[], + fix_funcs=[], +) + +emergency_contact_missing_task = BaseTask( + name="missing", + description="No emergency contact on file", + init_funcs=[init_emergency_contact_missing], + fix_funcs=[fix_update_emergency_contact], +) + +emergency_contact_outdated_task = BaseTask( + name="outdated", + description="Emergency contact information outdated", + init_funcs=[init_emergency_contact_outdated], + fix_funcs=[fix_update_emergency_contact], +) + +no_instructions_needed_task = BaseTask( + name="no_instructions_needed", + description="No special instructions required", + init_funcs=[], + fix_funcs=[], +) + +medication_instructions_task = BaseTask( + name="medication_instructions", + description="Medication instructions need acknowledgment", + init_funcs=[init_medication_instructions], + fix_funcs=[fix_acknowledge_medication], +) + +post_care_instructions_task = BaseTask( + name="post_care_instructions", + description="Post-care instructions need acknowledgment", + init_funcs=[init_post_care_instructions], + fix_funcs=[fix_acknowledge_post_care], +) + +pre_surgery_instructions_task = BaseTask( + name="pre_surgery_instructions", + description="Pre-surgery instructions need acknowledgment", + init_funcs=[init_pre_surgery_instructions], + fix_funcs=[fix_acknowledge_pre_surgery], +) + + +### SelectionSets + +consent_issues = SelectionSet( + tasks=[ + telehealth_consent_needed_task, + data_sharing_consent_needed_task, + ] +) + +emergency_contact_issues = SelectionSet( + tasks=[ + emergency_contact_missing_task, + emergency_contact_outdated_task, + ] +) + +instruction_acknowledgment_issues = SelectionSet( + tasks=[ + medication_instructions_task, + post_care_instructions_task, + pre_surgery_instructions_task, + ] +) + +telehealth_setup_selection_sets = [ + consent_issues, + emergency_contact_issues, + instruction_acknowledgment_issues, +] diff --git a/src/tau2/domains/healthcare/tasks/test_results_issues.py b/src/tau2/domains/healthcare/tasks/test_results_issues.py new file mode 100644 index 00000000..035453fe --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/test_results_issues.py @@ -0,0 +1,406 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### Init Functions + + +def init_test_results_ready(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Test results are ready and available.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "ready" + env.tools.db.test_results[ + "test_001" + ].result = "Complete Blood Count (CBC) - All values within normal range" + env.tools.db.test_results[ + "test_001" + ].notes = "Results reviewed and appear normal" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": True, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "Complete Blood Count (CBC) - All values within normal range", + "notes": "Results reviewed and appear normal", + } + ], + "messages_count": 0, + "outstanding_balance": 0, + }, + ) + ] + + +def init_test_results_pending(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Test results are still pending from lab.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "pending" + env.tools.db.test_results["test_001"].result = None + env.tools.db.test_results["test_001"].notes = None + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_test_results_waiting", # For pending results (non-fixable) + "patient_id": "patient_001", + "reason": "Pending test results - awaiting lab", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": False, + "test_results": [], # No results yet - still pending + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +def init_test_results_reviewed_by_doctor( + env: HealthcareEnvironment, +) -> list[EnvFunctionCall]: + """Test results ready but awaiting doctor review before release.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "reviewed" + env.tools.db.test_results[ + "test_001" + ].result = "Lipid Panel - Elevated cholesterol levels noted" + env.tools.db.test_results[ + "test_001" + ].notes = "Doctor review required before release - follow-up needed" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_test_results_doctor_review", # For reviewed results (non-fixable) + "patient_id": "patient_001", + "reason": "Pending test results - awaiting doctor review", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": False, # Not released to patient yet + "test_results": [], # Not released to patient - awaiting doctor review + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +### Fix Functions + + +def fix_provide_test_results_with_followup( + env: HealthcareEnvironment, +) -> list[ToolCall]: + """Provide ready test results to patient and schedule routine follow-up.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="check_test_results", + arguments={"patient_id": "patient_001", "test_id": "test_001"}, + ), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={"patient_id": "patient_001", "procedure_type": "follow_up"}, + ), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + compare_args=["doctor_id"], + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-20", + "time": "14:00", + "reason": "Routine follow-up to review test results and annual wellness check", + }, + compare_args=["patient_id", "doctor_id", "appointment_type"], + ), + ] + + +def init_results_normal(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Test results are normal.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "ready" + env.tools.db.test_results[ + "test_001" + ].result = "All test values within normal reference ranges" + env.tools.db.test_results["test_001"].notes = "No abnormalities detected" + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_test_results_routine_followup", + "patient_id": "patient_001", + "reason": "Pending routine follow-up for normal test results", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": True, + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "All test values within normal reference ranges", + "notes": "No abnormalities detected", + } + ], + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +def init_results_abnormal_minor(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Test results show minor abnormalities requiring follow-up.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "ready" + env.tools.db.test_results[ + "test_001" + ].result = "Slightly elevated cholesterol (220 mg/dL) - recommend dietary modifications" + env.tools.db.test_results[ + "test_001" + ].notes = "Minor abnormality - follow-up in 3 months recommended" + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_test_results_followup", # For scheduling follow-up + "patient_id": "patient_001", + "reason": "Pending test results follow-up - minor abnormalities", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": True, # Results are ready and available to patient + "test_results": [ + { + "test_name": "HbA1c (Diabetes screening)", + "test_date": "2024-05-10", + "result": "Slightly elevated cholesterol (220 mg/dL) - recommend dietary modifications", + "notes": "Minor abnormality - follow-up in 3 months recommended", + } + ], + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +def init_results_abnormal_critical(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Test results show critical abnormalities requiring immediate action.""" + if "test_001" in env.tools.db.test_results: + env.tools.db.test_results["test_001"].status = "reviewed" + env.tools.db.test_results[ + "test_001" + ].result = "CRITICAL: Severely elevated glucose (450 mg/dL) and abnormal kidney function" + env.tools.db.test_results[ + "test_001" + ].notes = "URGENT - Patient requires immediate medical evaluation" + + return [ + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_test_results_critical_transfer", # For emergency transfer + "patient_id": "patient_001", + "reason": "Pending test results - critical findings requiring immediate attention", + }, + ), + EnvFunctionCall( + env_type="user", + func_name="set_portal_info", + arguments={ + "upcoming_appointments": [], + "recent_visits": [], + "test_results_available": False, # Critical results not released to patient, needs nurse review + "test_results": [], # Critical results not released - requires nurse review + "messages_count": 0, + "outstanding_balance": 0, + }, + ), + ] + + +def fix_schedule_followup_minor(env: HealthcareEnvironment) -> list[ToolCall]: + """Schedule follow-up appointment for minor abnormalities.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="check_test_results", + arguments={"patient_id": "patient_001", "test_id": "test_001"}, + ), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={"patient_id": "patient_001", "procedure_type": "follow_up"}, + ), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-30"}, + compare_args=["doctor_id"], + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "follow_up", + "date": "2024-05-30", + "time": "10:00", + "reason": "Follow-up for abnormal test results - discuss findings and treatment plan", + }, + ), + ] + + +def fix_escalate_critical_results(env: HealthcareEnvironment) -> list[ToolCall]: + """Escalate critical test results to clinical staff.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="check_test_results", + arguments={"patient_id": "patient_001", "test_id": "test_001"}, + ), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def fix_inform_test_status(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient and check test results status.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall( + requestor="assistant", + name="check_test_results", + arguments={"patient_id": "patient_001", "test_id": "test_001"}, + ), + ] + + +### Base Tasks + +test_results_ready_task = BaseTask( + name="ready", + description="Test results ready and available", + init_funcs=[init_test_results_ready], + fix_funcs=[fix_inform_test_status], +) + +test_results_pending_task = BaseTask( + name="pending", + description="Test results still pending from lab", + init_funcs=[init_test_results_pending], + fix_funcs=[fix_inform_test_status], +) + +test_results_reviewed_by_doctor_task = BaseTask( + name="reviewed_by_doctor", + description="Results awaiting doctor review before release", + init_funcs=[init_test_results_reviewed_by_doctor], + fix_funcs=[fix_inform_test_status], +) + +results_normal_task = BaseTask( + name="normal", + description="Test results normal", + init_funcs=[init_results_normal], + fix_funcs=[fix_provide_test_results_with_followup], +) + +results_abnormal_minor_task = BaseTask( + name="abnormal_minor", + description="Minor abnormalities requiring follow-up", + init_funcs=[init_results_abnormal_minor], + fix_funcs=[fix_schedule_followup_minor], +) + +results_abnormal_critical_task = BaseTask( + name="abnormal_critical", + description="Critical abnormalities requiring immediate action", + init_funcs=[init_results_abnormal_critical], + fix_funcs=[fix_escalate_critical_results], +) + + +### SelectionSets + +abnormal_results_issues = SelectionSet( + tasks=[ + results_normal_task, + results_abnormal_minor_task, + results_abnormal_critical_task, + ] +) + +test_results_selection_sets = [ + abnormal_results_issues, +] diff --git a/src/tau2/domains/healthcare/tasks/urgent_triage_issues.py b/src/tau2/domains/healthcare/tasks/urgent_triage_issues.py new file mode 100644 index 00000000..03940a1f --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/urgent_triage_issues.py @@ -0,0 +1,390 @@ +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvFunctionCall +from tau2.domains.healthcare.environment import HealthcareEnvironment +from tau2.domains.healthcare.tasks.utils import BaseTask, SelectionSet + + +### Init Functions + + +def init_no_fever(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Temperature normal.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 98.6}, + ) + ] + + +def init_mild_fever(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Mild fever 99-101°F.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 100.2}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Mild fever", + "severity": "mild", + "duration": "1 day", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - mild fever", + }, + ), + ] + + +def init_high_fever(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """High fever 101-103°F.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 102.1}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "High fever with chills", + "severity": "moderate", + "duration": "2 days", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - high fever", + }, + ), + ] + + +def init_very_high_fever(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Very high fever >103°F (urgent).""" + return [ + EnvFunctionCall( + env_type="user", + func_name="set_temperature", + arguments={"temperature": 103.8}, + ), + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Very high fever with severe chills", + "severity": "severe", + "duration": "6 hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - very high fever", + }, + ), + ] + + +### Fix Functions + + +def fix_urgent_fever(env: HealthcareEnvironment) -> list[ToolCall]: + """Book urgent appointment for high fever.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="take_temperature", arguments={}), + ToolCall( + requestor="assistant", + name="verify_insurance_coverage", + arguments={"patient_id": "patient_001", "procedure_type": "urgent_care"}, + compare_args=["patient_id"], + ), + ToolCall( + requestor="assistant", + name="check_available_time_slots", + arguments={"doctor_id": "doc_001", "date": "2024-05-20"}, + ), + ToolCall(requestor="user", name="check_calendar", arguments={}), + ToolCall( + requestor="assistant", + name="book_appointment", + arguments={ + "patient_id": "patient_001", + "doctor_id": "doc_001", + "appointment_type": "urgent_care", + "date": "2024-05-20", + "time": "16:00", + "reason": "High fever requiring urgent evaluation", + }, + ), + ] + + +def init_no_pain(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """No significant pain.""" + return [] + + +def init_mild_pain(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Mild pain 1-3/10.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Mild headache", + "severity": "mild", + "duration": "few hours", + }, + ) + ] + + +def init_moderate_pain(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Moderate pain 4-6/10.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Moderate body aches and muscle pain", + "severity": "moderate", + "duration": "1 day", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_triage", + "patient_id": "patient_001", + "reason": "Pending triage - moderate pain", + }, + ), + ] + + +def init_severe_pain(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Severe pain 7-10/10 (urgent).""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Severe chest pain radiating to left arm", + "severity": "severe", + "duration": "30 minutes", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_transfer", + "patient_id": "patient_001", + "reason": "Pending triage - severe pain requiring transfer", + }, + ), + ] + + +def fix_urgent_pain(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for severe pain triage.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_symptoms", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def fix_very_high_fever(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for very high fever (≥103°F).""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="take_temperature", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def init_normal_breathing(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """No breathing issues.""" + return [] + + +def init_difficulty_breathing(env: HealthcareEnvironment) -> list[EnvFunctionCall]: + """Shortness of breath.""" + return [ + EnvFunctionCall( + env_type="user", + func_name="add_symptom", + arguments={ + "description": "Difficulty breathing, shortness of breath", + "severity": "severe", + "duration": "2 hours", + }, + ), + EnvFunctionCall( + env_type="assistant", + func_name="create_appointment_marker", + arguments={ + "appointment_id": "pending_urgent_transfer", + "patient_id": "patient_001", + "reason": "Pending triage - difficulty breathing requiring transfer", + }, + ), + ] + + +def fix_breathing_emergency(env: HealthcareEnvironment) -> list[ToolCall]: + """Transfer to nurse for breathing difficulty.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_symptoms", arguments={}), + ToolCall(requestor="assistant", name="transfer_to_nurse", arguments={}), + ] + + +def fix_no_urgent_symptoms(env: HealthcareEnvironment) -> list[ToolCall]: + """Verify patient and symptoms, confirm no urgent care needed.""" + return [ + ToolCall( + requestor="assistant", + name="get_patient_details", + arguments={"full_name": "Sarah Johnson", "date_of_birth": "1985-03-15"}, + ), + ToolCall(requestor="user", name="check_symptoms", arguments={}), + ] + + +### Base Tasks + +no_fever_task = BaseTask( + name="no_fever", + description="Temperature normal", + init_funcs=[init_no_fever], + fix_funcs=[fix_no_urgent_symptoms], +) + +mild_fever_task = BaseTask( + name="mild_fever", + description="Mild fever 99-101°F", + init_funcs=[init_mild_fever], + fix_funcs=[fix_urgent_fever], +) + +high_fever_task = BaseTask( + name="high_fever", + description="High fever 101-103°F", + init_funcs=[init_high_fever], + fix_funcs=[fix_urgent_fever], +) + +very_high_fever_task = BaseTask( + name="very_high_fever", + description="Very high fever >103°F", + init_funcs=[init_very_high_fever], + fix_funcs=[fix_very_high_fever], +) + +no_pain_task = BaseTask( + name="no_pain", + description="No significant pain", + init_funcs=[init_no_pain], + fix_funcs=[fix_no_urgent_symptoms], +) + +mild_pain_task = BaseTask( + name="mild_pain", + description="Mild pain 1-3/10", + init_funcs=[init_mild_pain], + fix_funcs=[fix_no_urgent_symptoms], +) + +moderate_pain_task = BaseTask( + name="moderate_pain", + description="Moderate pain 4-6/10", + init_funcs=[init_moderate_pain], + fix_funcs=[fix_urgent_fever], +) + +severe_pain_task = BaseTask( + name="severe_pain", + description="Severe pain 7-10/10", + init_funcs=[init_severe_pain], + fix_funcs=[fix_urgent_pain], +) + +normal_breathing_task = BaseTask( + name="normal_breathing", + description="No breathing issues", + init_funcs=[init_normal_breathing], + fix_funcs=[fix_no_urgent_symptoms], +) + +difficulty_breathing_task = BaseTask( + name="difficulty_breathing", + description="Shortness of breath", + init_funcs=[init_difficulty_breathing], + fix_funcs=[fix_breathing_emergency], +) + + +### SelectionSets + +fever_level_issues = SelectionSet( + tasks=[ + mild_fever_task, + high_fever_task, + very_high_fever_task, + ] +) + +pain_severity_issues = SelectionSet( + tasks=[ + moderate_pain_task, + ] +) + +breathing_issues = SelectionSet(tasks=[]) + +urgent_triage_selection_sets = [ + fever_level_issues, + pain_severity_issues, + breathing_issues, +] diff --git a/src/tau2/domains/healthcare/tasks/utils.py b/src/tau2/domains/healthcare/tasks/utils.py new file mode 100644 index 00000000..9c4fdd41 --- /dev/null +++ b/src/tau2/domains/healthcare/tasks/utils.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import re +from itertools import product +from typing import Callable, Optional + +from pydantic import BaseModel, Field + +from tau2.data_model.message import ToolCall +from tau2.data_model.tasks import EnvAssertion, EnvFunctionCall +from tau2.environment.environment import Environment + +InitFuncType = Callable[[Environment], list[EnvFunctionCall | EnvAssertion]] +FixFuncType = Callable[[Environment], list[ToolCall]] | None +EnvAssertionType = Callable[[Environment], list[EnvAssertion]] + + +class BaseTask(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + name: str + description: str + init_funcs: list[InitFuncType] + fix_funcs: list[FixFuncType] + extra_env_assertions: list[EnvAssertionType] = Field(default_factory=list) + + +class SelectionSet(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + tasks: list[BaseTask] + + +class ComposedTask(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + name: str + description: str + composed_from: list[BaseTask] + init_funcs: list[InitFuncType] + fix_funcs: list[FixFuncType] + extra_env_assertions: list[EnvAssertionType] = Field(default_factory=list) + + def __str__(self): + lines = [] + lines.append("-" * len(self.name)) + lines.append(self.name) + lines.append("-" * len(self.name)) + lines.append(f"Description: {self.description}") + lines.append("Base Tasks:") + for task in self.composed_from: + lines.append(f" - {task.name}: {task.description}") + lines.append("Init Funcs:") + for func in self.init_funcs: + lines.append(f" - {func.__name__}") + lines.append("Fix Funcs:") + for func in self.fix_funcs: + func_name = func.__name__ if func is not None else "None" + lines.append(f" - {func_name}") + lines.append("Extra Env Assertions:") + for func in self.extra_env_assertions: + lines.append(f" - {func.__name__}") + + return "\n".join(lines) + + def __repr__(self): + return self.__str__() + + +def compose_tasks( + selection_sets: list[SelectionSet], + task_validator: Optional[Callable[[list[Optional[BaseTask]]], bool]] = None, +) -> list[ComposedTask]: + """ + Return all the combinations of selecting 0 or more tasks from the selection sets + """ + + product_tasks = list( + product(*[selection_set.tasks + [None] for selection_set in selection_sets]) + ) + composed_tasks = [] + for tasks in product_tasks: + if task_validator is not None: + if not task_validator(list(tasks)): + continue + tasks = sorted([t for t in tasks if t is not None], key=lambda x: x.name) + if task_validator is None and len(tasks) == 0: + continue + init_funcs = [f for t in tasks for f in t.init_funcs] + seen_fix = set() + fix_funcs = [] + for t in tasks: + for f in t.fix_funcs: + func_id = id(f) if f is not None else None + if func_id not in seen_fix: + seen_fix.add(func_id) + fix_funcs.append(f) + extra_env_assertions = [f for t in tasks for f in t.extra_env_assertions] + composed_task = ComposedTask( + name="|".join([t.name for t in tasks]), + description=", ".join([t.description for t in tasks]), + composed_from=tasks, + init_funcs=init_funcs, + fix_funcs=fix_funcs, + extra_env_assertions=extra_env_assertions, + ) + composed_tasks.append(composed_task) + return composed_tasks + + +def get_intent_from_task_id(task_id: str) -> str: + """ + Extract the intent from the task_id. + task_id is of the form: [intent]action1|action2|...|actionk[PERSONA:persona] + """ + pat = r"^\[([a-zA-Z_]+)\]" + match = re.search(pat, task_id) + if match: + return match.group(1) + else: + raise ValueError(f"Could not extract intent from task_id: {task_id}") + + +def get_persona_from_task_id(task_id: str) -> str: + """ + Extract the persona from the task_id. + task_id is of the form: [intent]action1|action2|...|actionk[PERSONA:persona] + """ + pat = r"\[PERSONA:([a-zA-Z_]+)\]" + match = re.search(pat, task_id) + if match: + return match.group(1) + else: + raise ValueError(f"Could not extract intent from task_id: {task_id}") + + +def get_num_issues_from_task_id(task_id: str) -> int: + """ + Extract the number of issues from the task_id. + task_id is of the form: [intent]action1|action2|...|actionk[PERSONA:persona] + """ + return len(task_id.split("|")) diff --git a/src/tau2/domains/healthcare/tools.py b/src/tau2/domains/healthcare/tools.py new file mode 100644 index 00000000..334f29b9 --- /dev/null +++ b/src/tau2/domains/healthcare/tools.py @@ -0,0 +1,866 @@ +from datetime import datetime +from typing import Any, Dict, List, Optional + +from loguru import logger + +from tau2.domains.healthcare.data_model import ( + Appointment, + AppointmentStatus, + AppointmentType, + Doctor, + HealthcareDB, + Patient, + Payment, + Prescription, + PrescriptionStatus, + TestResult, +) +from tau2.environment.toolkit import ToolKitBase, ToolType, is_tool + + +class HealthcareTools(ToolKitBase): + """All the tools for the healthcare domain (agent-side).""" + + db: HealthcareDB + + def __init__(self, db: HealthcareDB) -> None: + super().__init__(db) + + # Helper methods + + def _get_patient(self, patient_id: str) -> Patient: + """Get patient from database.""" + if patient_id not in self.db.patients: + raise ValueError(f"Patient {patient_id} not found") + return self.db.patients[patient_id] + + def _find_patient_by_identity( + self, full_name: str, date_of_birth: str + ) -> Optional[Patient]: + """Find patient by full name and date of birth.""" + for patient in self.db.patients.values(): + patient_full_name = f"{patient.name.first_name} {patient.name.last_name}" + if ( + patient_full_name == full_name + and patient.date_of_birth == date_of_birth + ): + return patient + return None + + def _get_doctor(self, doctor_id: str) -> Doctor: + """Get doctor from database.""" + if doctor_id not in self.db.doctors: + raise ValueError(f"Doctor {doctor_id} not found") + return self.db.doctors[doctor_id] + + def _get_appointment(self, appointment_id: str) -> Appointment: + """Get appointment from database.""" + if appointment_id not in self.db.appointments: + raise ValueError(f"Appointment {appointment_id} not found") + return self.db.appointments[appointment_id] + + def _get_prescription(self, prescription_id: str) -> Prescription: + """Get prescription from database.""" + if prescription_id not in self.db.prescriptions: + raise ValueError(f"Prescription {prescription_id} not found") + return self.db.prescriptions[prescription_id] + + def _get_new_appointment_id(self) -> str: + """Generate a new appointment ID.""" + for i in range(1, 11): + apt_id = f"APPT_NEW_{i:03d}" + if apt_id not in self.db.appointments: + return apt_id + raise ValueError("Too many appointments created") + + def _get_new_payment_id(self) -> str: + """Generate a new payment ID.""" + for i in range(1, 11): + pay_id = f"PAY_NEW_{i:03d}" + if pay_id not in self.db.payments: + return pay_id + raise ValueError("Too many payments created") + + def _get_current_datetime(self) -> str: + """Get current datetime (fixed for simulation).""" + return "2024-05-15T15:00:00" + + def _is_time_slot_available(self, doctor: Doctor, date: str, time: str) -> bool: + """Check if a doctor has a specific time slot available.""" + date_obj = datetime.strptime(date, "%Y-%m-%d") + day_name = date_obj.strftime("%A") + + if day_name not in doctor.available_days: + return False + + if time not in doctor.available_times: + return False + + for apt in self.db.appointments.values(): + if ( + apt.doctor_id == doctor.doctor_id + and apt.date == date + and apt.time == time + and apt.status == "scheduled" + ): + return False + + return True + + @is_tool(ToolType.READ) + def get_patient_details(self, full_name: str, date_of_birth: str) -> Patient: + """ + Retrieve complete patient information. + + Args: + full_name: Patient's full name (e.g., "Sarah Johnson") + date_of_birth: Patient's date of birth in YYYY-MM-DD format (e.g., "1985-03-15") + + Returns: + Complete patient record with all details + """ + patient = self._find_patient_by_identity(full_name, date_of_birth) + if patient is None: + raise ValueError( + f"No patient found with name '{full_name}' and date of birth '{date_of_birth}'. Please verify the patient's identity information." + ) + logger.info( + f"Retrieved patient details for {full_name} (patient_id: {patient.patient_id})" + ) + return patient + + @is_tool(ToolType.READ) + def get_appointment_details(self, appointment_id: str) -> Appointment: + """ + Retrieve details of a specific appointment. + + Args: + appointment_id: The unique identifier for the appointment + + Returns: + Complete appointment information + """ + return self._get_appointment(appointment_id) + + @is_tool(ToolType.READ) + def search_appointments( + self, + patient_id: str, + status: Optional[AppointmentStatus] = None, + ) -> List[Appointment]: + """ + Search for appointments for a specific patient, optionally filtered by status. + + Args: + patient_id: The patient ID to search appointments for + status: Optional status filter (scheduled, completed, cancelled, no_show) + + Returns: + List of matching appointments + """ + patient = self._get_patient(patient_id) + results = [] + + for apt_id in patient.appointment_ids: + if apt_id in self.db.appointments: + apt = self.db.appointments[apt_id] + if status is None or apt.status == status: + results.append(apt) + + return results + + @is_tool(ToolType.READ) + def list_available_doctors( + self, + specialty: Optional[str] = None, + date: Optional[str] = None, + ) -> List[Doctor]: + """ + List all doctors, optionally filtered by specialty and availability on a specific date. + + Args: + specialty: Optional specialty to filter by (e.g., "General Practice", "Cardiology") + date: Optional date in YYYY-MM-DD format to check availability + + Returns: + List of doctors matching the criteria + """ + results = [] + + for doctor in self.db.doctors.values(): + if specialty and doctor.specialty.lower() != specialty.lower(): + continue + + if date: + date_obj = datetime.strptime(date, "%Y-%m-%d") + day_name = date_obj.strftime("%A") + if day_name not in doctor.available_days: + continue + + results.append(doctor) + + return results + + @is_tool(ToolType.READ) + def check_available_time_slots( + self, + doctor_id: str, + date: str, + ) -> List[str]: + """ + Check what time slots are available for a specific doctor on a specific date. + + Args: + doctor_id: The doctor's unique identifier + date: Date to check in YYYY-MM-DD format + + Returns: + List of available time slots in HH:MM format + """ + doctor = self._get_doctor(doctor_id) + date_obj = datetime.strptime(date, "%Y-%m-%d") + day_name = date_obj.strftime("%A") + + if day_name not in doctor.available_days: + return [] + + available_slots = [] + for time_slot in doctor.available_times: + if self._is_time_slot_available(doctor, date, time_slot): + available_slots.append(time_slot) + + return available_slots + + @is_tool(ToolType.WRITE) + def book_appointment( + self, + patient_id: str, + doctor_id: str, + appointment_type: AppointmentType, + date: str, + time: str, + reason: str, + ) -> Appointment: + """ + Book a new appointment for a patient. + + Args: + patient_id: The patient's unique identifier + doctor_id: The doctor's unique identifier + appointment_type: Type of appointment (routine_checkup, follow_up, urgent_care, specialist) + date: Appointment date in YYYY-MM-DD format + time: Appointment time in HH:MM format (24-hour) + reason: Reason for the visit + + Returns: + The newly created appointment + """ + patient = self._get_patient(patient_id) + doctor = self._get_doctor(doctor_id) + + if not self._is_time_slot_available(doctor, date, time): + raise ValueError( + f"Time slot {time} on {date} is not available for Dr. {doctor.name.last_name}" + ) + + base_costs = { + "routine_checkup": 150, + "follow_up": 100, + "urgent_care": 200, + "specialist": 250, + } + cost = base_costs.get(appointment_type, 150) + + apt_id = self._get_new_appointment_id() + appointment = Appointment( + appointment_id=apt_id, + patient_id=patient_id, + doctor_id=doctor_id, + appointment_type=appointment_type, + date=date, + time=time, + status="scheduled", + reason=reason, + notes=None, + created_at=self._get_current_datetime(), + cost=cost, + ) + + self.db.appointments[apt_id] = appointment + patient.appointment_ids.append(apt_id) + + logger.info(f"Booked appointment {apt_id} for patient {patient_id}") + return appointment + + @is_tool(ToolType.WRITE) + def cancel_appointment( + self, + appointment_id: str, + reason: str, + ) -> Appointment: + """ + Cancel an existing appointment. + + Args: + appointment_id: The appointment ID to cancel + reason: Reason for cancellation + + Returns: + The updated appointment with cancelled status + """ + appointment = self._get_appointment(appointment_id) + + if appointment.status in ["cancelled", "completed"]: + raise ValueError( + f"Cannot cancel appointment with status: {appointment.status}" + ) + + appointment.status = "cancelled" + if appointment.notes: + appointment.notes += f" | Cancelled: {reason}" + else: + appointment.notes = f"Cancelled: {reason}" + + logger.info(f"Cancelled appointment {appointment_id}") + return appointment + + @is_tool(ToolType.WRITE) + def reschedule_appointment( + self, + appointment_id: str, + new_date: str, + new_time: str, + ) -> Appointment: + """ + Reschedule an existing appointment to a new date and time. + + Args: + appointment_id: The appointment ID to reschedule + new_date: New date in YYYY-MM-DD format + new_time: New time in HH:MM format + + Returns: + The updated appointment + """ + appointment = self._get_appointment(appointment_id) + + if appointment.status != "scheduled": + raise ValueError( + f"Cannot reschedule appointment with status: {appointment.status}" + ) + + doctor = self._get_doctor(appointment.doctor_id) + if not self._is_time_slot_available(doctor, new_date, new_time): + raise ValueError(f"Time slot {new_time} on {new_date} is not available") + + old_date = appointment.date + old_time = appointment.time + appointment.date = new_date + appointment.time = new_time + if appointment.notes: + appointment.notes += f" | Rescheduled from {old_date} {old_time}" + else: + appointment.notes = f"Rescheduled from {old_date} {old_time}" + + logger.info( + f"Rescheduled appointment {appointment_id} to {new_date} {new_time}" + ) + return appointment + + @is_tool(ToolType.READ) + def verify_insurance_coverage( + self, + patient_id: str, + procedure_type: Optional[str] = None, + ) -> dict: + """ + Verify patient's insurance coverage and copay information. + + Args: + patient_id: The patient's unique identifier + procedure_type: Optional specific procedure to check coverage for + + Returns: + Dictionary with insurance verification details + """ + patient = self._get_patient(patient_id) + insurance = patient.insurance + + result = { + "verified": True, + "provider": insurance.provider, + "policy_number": insurance.policy_number, + "copay_amount": insurance.copay_amount, + "coverage_details": insurance.coverage_details, + } + + if procedure_type: + result["procedure_covered"] = ( + "routine" in insurance.coverage_details.lower() + ) + + return result + + @is_tool(ToolType.READ) + def get_prescription_details(self, prescription_id: str) -> Prescription: + """ + Get details of a specific prescription. + + Args: + prescription_id: The prescription's unique identifier + + Returns: + Complete prescription information + """ + return self._get_prescription(prescription_id) + + @is_tool(ToolType.WRITE) + def request_prescription_refill( + self, + prescription_id: str, + patient_id: str, + ) -> Prescription: + """ + Request a refill for an existing prescription. Checks if refills are available. + + Args: + prescription_id: The prescription ID to refill + patient_id: The patient's ID (for verification) + + Returns: + Updated prescription information + """ + prescription = self._get_prescription(prescription_id) + + if prescription.patient_id != patient_id: + raise ValueError( + f"Prescription {prescription_id} does not belong to patient {patient_id}" + ) + + if prescription.status != "active": + raise ValueError( + f"Cannot refill prescription with status: {prescription.status}" + ) + + if prescription.refills_remaining <= 0: + raise ValueError( + "No refills remaining. Patient needs to contact doctor for new prescription." + ) + + prescription.refills_remaining -= 1 + + if prescription.refills_remaining == 0: + prescription.status = "refill_needed" + + logger.info(f"Processed refill for prescription {prescription_id}") + return prescription + + @is_tool(ToolType.READ) + def check_test_results( + self, + patient_id: str, + test_id: Optional[str] = None, + ) -> List[TestResult]: + """ + Check test results for a patient. If test_id provided, returns that specific test. + + Args: + patient_id: The patient's unique identifier + test_id: Optional specific test ID to retrieve + + Returns: + List of test results (or single test if test_id provided) + """ + if test_id: + test = self.db.test_results.get(test_id) + if not test: + raise ValueError(f"Test {test_id} not found") + if test.patient_id != patient_id: + raise ValueError( + f"Test {test_id} does not belong to patient {patient_id}" + ) + return [test] + + results = [] + for test in self.db.test_results.values(): + if test.patient_id == patient_id: + results.append(test) + + return results + + @is_tool(ToolType.GENERIC) + def calculate_cost( + self, + appointment_type: AppointmentType, + insurance_provider: str, + ) -> dict: + """ + Calculate the estimated cost for an appointment including insurance copay. + + Args: + appointment_type: Type of appointment + insurance_provider: Patient's insurance provider + + Returns: + Dictionary with cost breakdown + """ + base_costs = { + "routine_checkup": 150, + "follow_up": 100, + "urgent_care": 200, + "specialist": 250, + } + + copay_amounts = { + "BlueCross": 20, + "Aetna": 25, + "UnitedHealth": 20, + "Medicare": 0, + "Medicaid": 0, + "SelfPay": 0, + } + + base_cost = base_costs.get(appointment_type, 150) + copay = copay_amounts.get(insurance_provider, 30) + + return { + "base_cost": base_cost, + "copay": copay if insurance_provider != "SelfPay" else base_cost, + "insurance_covers": base_cost - copay + if insurance_provider != "SelfPay" + else 0, + "patient_pays": copay if insurance_provider != "SelfPay" else base_cost, + } + + @is_tool(ToolType.GENERIC) + def transfer_to_nurse(self) -> str: + """ + Transfer the patient to a nurse for clinical questions or triage. + + Returns: + Transfer confirmation message + """ + logger.info("Transferring patient to nurse") + return "Transferring you to a nurse who can better assist with your clinical questions. Please hold." + + @is_tool(ToolType.GENERIC) + def transfer_to_human_agent(self) -> str: + """ + Transfer the patient to a human agent when the request cannot be handled automatically. + + Returns: + Transfer confirmation message + """ + logger.info("Transferring patient to human agent") + return "I'm transferring you to a specialist who can better assist you. Please hold." + + def create_appointment_for_test( + self, + appointment_id: str, + patient_id: str, + doctor_id: str, + appointment_type: str, + date: str, + time: str, + status: str, + reason: str, + cost: int, + notes: Optional[str] = None, + ) -> None: + """Create an appointment for test scenario initialization.""" + from datetime import datetime + from tau2.domains.healthcare.data_model import Appointment + + appointment = Appointment( + appointment_id=appointment_id, + patient_id=patient_id, + doctor_id=doctor_id, + appointment_type=appointment_type, # type: ignore + date=date, + time=time, + status=status, # type: ignore + reason=reason, + notes=notes, + created_at=datetime.now().isoformat(), + cost=cost, + ) + + self.db.appointments[appointment_id] = appointment + + patient = self.db.patients.get(patient_id) + if patient: + if patient.appointment_ids is None: + patient.appointment_ids = [] + patient.appointment_ids.append(appointment_id) + + logger.info( + f"Created test appointment: {appointment_id} for patient {patient_id}" + ) + + def assert_appointment_exists( + self, patient_id: str, appointment_type: Optional[str] = None + ) -> bool: + """Assert that an appointment exists for the patient.""" + patient = self.db.patients.get(patient_id) + if not patient: + return False + + if not patient.appointment_ids: + return False + + if appointment_type is None: + return True + + for apt_id in patient.appointment_ids: + apt = self.db.appointments.get(apt_id) + if apt and apt.appointment_type == appointment_type: + return True + + return False + + def assert_appointment_status( + self, appointment_id: str, expected_status: str + ) -> bool: + """Assert an appointment's status (scheduled, cancelled, completed).""" + apt = self.db.appointments.get(appointment_id) + if not apt: + return False + return apt.status == expected_status + + def assert_prescription_refills_remaining( + self, prescription_id: str, expected_count: int + ) -> bool: + """Assert the number of refills remaining on a prescription.""" + rx = self.db.prescriptions.get(prescription_id) + if not rx: + return False + return rx.refills_remaining == expected_count + + def assert_prescription_status( + self, prescription_id: str, expected_status: str + ) -> bool: + """Assert a prescription's status (active, expired, discontinued).""" + rx = self.db.prescriptions.get(prescription_id) + if not rx: + return False + return rx.status == expected_status + + def assert_patient_has_insurance(self, patient_id: str, expected: bool) -> bool: + """Assert whether a patient has insurance information on file.""" + patient = self.db.patients.get(patient_id) + if not patient: + return False + has_insurance = patient.insurance is not None + return has_insurance == expected + + def assert_insurance_provider( + self, patient_id: str, expected_provider: str + ) -> bool: + """Assert the patient's insurance provider.""" + patient = self.db.patients.get(patient_id) + if not patient or not patient.insurance: + return False + return patient.insurance.provider == expected_provider + + def assert_appointment_count_exceeds_baseline(self) -> bool: + """Assert that real appointments exceed baseline + markers.""" + num_markers = sum( + 1 + for apt in self.db.appointments.values() + if apt.date == "2024-01-01" and apt.time == "00:00" + ) + num_real = len(self.db.appointments) - num_markers + + if num_markers == 0: + return True + + return num_real >= (2 + num_markers) + + def assert_tool_was_called(self, tool_name: str) -> bool: + """ + Verify that a specific tool was called during the conversation. + + Args: + tool_name: Name of the tool to check (e.g., "get_prescription_details") + + Returns: + True if the tool was called at least once, False otherwise + """ + return tool_name in self.db.tool_call_history + + def assert_tool_was_not_called(self, tool_name: str) -> bool: + """ + Verify that a specific tool was NOT called during the conversation. + + Args: + tool_name: Name of the tool to check + + Returns: + True if the tool was never called, False if it was called + """ + return tool_name not in self.db.tool_call_history + + def set_prescription_refills(self, prescription_id: str, refills: int) -> None: + """Set the number of refills remaining on a prescription.""" + if prescription_id not in self.db.prescriptions: + raise ValueError(f"Prescription {prescription_id} not found") + self.db.prescriptions[prescription_id].refills_remaining = refills + + def set_prescription_status(self, prescription_id: str, status: str) -> None: + """Set the status of a prescription (active, expired, discontinued, refill_needed).""" + if prescription_id not in self.db.prescriptions: + raise ValueError(f"Prescription {prescription_id} not found") + self.db.prescriptions[prescription_id].status = status + + def set_prescription_medication( + self, prescription_id: str, medication_name: str, dosage: str + ) -> None: + """Set the medication name and dosage for a prescription.""" + if prescription_id not in self.db.prescriptions: + raise ValueError(f"Prescription {prescription_id} not found") + self.db.prescriptions[prescription_id].medication_name = medication_name + self.db.prescriptions[prescription_id].dosage = dosage + + def create_appointment_marker( + self, + appointment_id: str, + patient_id: str, + reason: str = "Pending booking request marker", + ) -> None: + """Create a temporary appointment marker to indicate pending booking request.""" + from tau2.domains.healthcare.data_model import Appointment + from datetime import datetime + + marker_appt = Appointment( + appointment_id=appointment_id, + patient_id=patient_id, + doctor_id="doc_001", + appointment_type="routine_checkup", + date="2024-01-01", # Placeholder date + time="00:00", + status="scheduled", + reason=reason, + created_at=datetime.now().isoformat(), + cost=0, + ) + self.db.appointments[appointment_id] = marker_appt + + @is_tool(ToolType.READ) + def get_vital_signs_history( + self, patient_id: str, days: int = 30 + ) -> List[Dict[str, Any]]: + """ + Retrieve vital signs history for the specified number of days. + + Args: + patient_id: Patient identifier + days: Number of days to look back (default 30) + + Returns: + List of vital signs measurements with timestamps + """ + from datetime import datetime, timedelta + + patient = self._get_patient(patient_id) + cutoff = datetime.now() - timedelta(days=days) + + recent_vitals = [ + vs + for vs in patient.vital_signs_history + if datetime.fromisoformat(vs.timestamp) > cutoff + ] + + result = [] + for vs in recent_vitals: + result.append( + { + "timestamp": vs.timestamp, + "blood_pressure": f"{vs.blood_pressure_systolic}/{vs.blood_pressure_diastolic}" + if vs.blood_pressure_systolic + else None, + "heart_rate": vs.heart_rate, + "temperature": vs.temperature, + "respiratory_rate": vs.respiratory_rate, + "oxygen_saturation": vs.oxygen_saturation, + "weight": vs.weight, + "height": vs.height, + } + ) + + logger.info(f"Retrieved {len(result)} vital signs for patient {patient_id}") + return result + + @is_tool(ToolType.READ) + def get_lab_results( + self, patient_id: str, test_type: Optional[str] = None + ) -> List[Dict[str, Any]]: + """ + Retrieve laboratory test results for a patient. + + Args: + patient_id: Patient identifier + test_type: Optional filter for specific test type (e.g., 'HbA1c', 'Lipid Panel') + + Returns: + List of lab results sorted by date (most recent first) + """ + patient = self._get_patient(patient_id) + + results = [ + self.db.lab_results[lab_id] + for lab_id in patient.lab_result_ids + if lab_id in self.db.lab_results + ] + + if test_type: + results = [r for r in results if r.test_type == test_type] + + results = sorted(results, key=lambda x: x.test_date, reverse=True) + + result = [] + for lab in results: + result.append( + { + "test_id": lab.test_id, + "test_type": lab.test_type, + "test_date": lab.test_date, + "results": lab.results, + "status": lab.status, + "critical": lab.critical, + "ordering_doctor": lab.ordering_doctor, + } + ) + + logger.info(f"Retrieved {len(result)} lab results for patient {patient_id}") + return result + + @is_tool(ToolType.READ) + def get_chronic_conditions(self, patient_id: str) -> List[Dict[str, Any]]: + """ + Get detailed information about patient's chronic medical conditions. + + Args: + patient_id: Patient identifier + + Returns: + List of chronic conditions with severity and control status + """ + patient = self._get_patient(patient_id) + + conditions = [] + for condition in patient.chronic_conditions: + conditions.append( + { + "condition_name": condition.condition_name, + "icd10_code": condition.icd10_code, + "diagnosed_date": condition.diagnosed_date, + "severity": condition.severity, + "controlled": condition.controlled, + "requires_monitoring": condition.requires_monitoring, + } + ) + + logger.info( + f"Retrieved {len(conditions)} chronic conditions for patient {patient_id}" + ) + return conditions diff --git a/src/tau2/domains/healthcare/user_data_model.py b/src/tau2/domains/healthcare/user_data_model.py new file mode 100644 index 00000000..ab3de02d --- /dev/null +++ b/src/tau2/domains/healthcare/user_data_model.py @@ -0,0 +1,231 @@ +from typing import List, Literal, Optional + +from pydantic import BaseModel, Field + +from tau2.environment.db import DB + +# Type definitions +SymptomSeverity = Literal["mild", "moderate", "severe"] + + +class InsuranceCard(BaseModel): + """Information visible on the patient's insurance card.""" + + provider: str = Field(description="Insurance provider name as shown on card") + policy_number: str = Field(description="Policy number on the card") + group_number: str = Field(description="Group number on the card") + member_name: str = Field(description="Member name on the card") + copay_info: str = Field(description="Copay information printed on card") + + +class Symptom(BaseModel): + """A symptom the patient is experiencing.""" + + description: str = Field(description="Description of the symptom") + severity: SymptomSeverity = Field(description="Severity level") + duration: str = Field(description="How long they've had this symptom") + + +class MedicationBottle(BaseModel): + """Information visible on a medication bottle at home.""" + + prescription_number: str = Field(description="Prescription number on bottle") + medication_name: str = Field(description="Name of medication") + dosage: str = Field(description="Dosage instructions on label") + refills_remaining: int = Field(description="Number of refills remaining on label") + prescribing_doctor: str = Field(description="Doctor name on bottle") + pharmacy_name: str = Field(description="Pharmacy name on bottle") + pharmacy_phone: str = Field(description="Pharmacy phone number") + + +class TimeSlot(BaseModel): + """A time slot in the patient's calendar.""" + + date: str = Field(description="Date in YYYY-MM-DD format") + time: str = Field(description="Time in HH:MM format") + available: bool = Field(description="Whether this slot is available") + reason: Optional[str] = Field(default=None, description="Reason if not available") + + +class PatientPortalInfo(BaseModel): + """Information accessible through the patient portal.""" + + upcoming_appointments: List[str] = Field( + default_factory=list, description="List of upcoming appointment descriptions" + ) + recent_visits: List[str] = Field( + default_factory=list, description="List of recent visit summaries" + ) + test_results_available: bool = Field( + default=False, description="Whether test results are available to view" + ) + test_results: List[dict] = Field( + default_factory=list, + description="List of test result summaries (test_name, test_date, result, notes)", + ) + messages_count: int = Field(default=0, description="Number of unread messages") + outstanding_balance: int = Field( + default=0, description="Outstanding balance in dollars" + ) + + +class BloodPressureReading(BaseModel): + """Blood pressure measurement from home monitor.""" + + systolic: int = Field(description="Systolic pressure in mmHg") + diastolic: int = Field(description="Diastolic pressure in mmHg") + + +class PainAssessment(BaseModel): + """PQRST pain assessment.""" + + provocation: str = Field(description="What makes it better or worse") + quality: str = Field(description="Description of pain type") + radiation: str = Field(description="Where the pain radiates") + severity: int = Field(description="Pain scale 0-10") + timing: str = Field(description="When it occurs and duration") + + +class PatientDevice(BaseModel): + """ + Represents physical items and information the patient has access to. + This is what the patient can check or use during the interaction. + """ + + insurance_card: InsuranceCard = Field(description="The patient's insurance card") + current_symptoms: List[Symptom] = Field( + default_factory=list, + description="Symptoms the patient is currently experiencing", + ) + current_temperature: Optional[float] = Field( + default=None, description="Current body temperature in Fahrenheit (if measured)" + ) + medications_at_home: List[MedicationBottle] = Field( + default_factory=list, description="Medication bottles the patient has at home" + ) + calendar_availability: List[TimeSlot] = Field( + default_factory=list, description="Patient's calendar availability" + ) + portal_info: Optional[PatientPortalInfo] = Field( + default=None, description="Information from patient portal (if logged in)" + ) + + # Patient actions and confirmations + confirmed_appointments: List[str] = Field( + default_factory=list, + description="List of appointment IDs patient has confirmed", + ) + consents_provided: List[str] = Field( + default_factory=list, description="List of consent types patient has provided" + ) + acknowledged_instructions: List[str] = Field( + default_factory=list, + description="List of instruction types patient has acknowledged", + ) + notification_preferences: List[str] = Field( + default_factory=list, description="List of enabled notification types" + ) + pharmacy_transfer_requests: List[dict] = Field( + default_factory=list, description="List of pharmacy transfer requests" + ) + + # Home medical monitoring devices + has_blood_pressure_monitor: bool = Field( + default=False, + description="Whether patient has a blood pressure monitor at home", + ) + latest_bp_reading: Optional[BloodPressureReading] = Field( + default=None, description="Most recent blood pressure reading from home monitor" + ) + has_glucose_meter: bool = Field( + default=False, description="Whether patient has a glucose meter at home" + ) + latest_glucose_reading: Optional[int] = Field( + default=None, description="Most recent blood glucose reading in mg/dL" + ) + glucose_measurement_time: Optional[str] = Field( + default=None, + description="When glucose was measured (e.g., 'Fasting (8am)', 'After meal')", + ) + has_pulse_oximeter: bool = Field( + default=False, description="Whether patient has a pulse oximeter at home" + ) + latest_spo2_reading: Optional[int] = Field( + default=None, description="Most recent oxygen saturation (SpO2) percentage" + ) + latest_heart_rate: Optional[int] = Field( + default=None, description="Most recent heart rate in beats per minute" + ) + current_pain: Optional[PainAssessment] = Field( + default=None, description="Current pain assessment using PQRST format" + ) + uploaded_photos: List[dict] = Field( + default_factory=list, + description="List of photos uploaded during telehealth sessions", + ) + + +class EmergencyContact(BaseModel): + """Emergency contact information.""" + + name: str = Field(description="Emergency contact's full name") + phone: str = Field(description="Emergency contact's phone number") + relationship: str = Field(description="Relationship to patient") + + +class PatientSurroundings(BaseModel): + """ + Context and environment around the patient during the interaction. + """ + + patient_id: str = Field(description="The patient's ID in the system") + full_name: str = Field(description="Patient's full name") + date_of_birth: str = Field( + description="Patient's date of birth (for identity verification)" + ) + location: Literal["home", "work", "on_the_go"] = Field( + default="home", description="Where the patient is calling from" + ) + has_internet_access: bool = Field( + default=True, description="Whether patient has internet access for portal" + ) + payment_methods_available: List[Literal["credit_card", "debit_card", "cash"]] = ( + Field( + default_factory=lambda: ["credit_card"], + description="Payment methods patient has available", + ) + ) + emergency_contact: Optional[EmergencyContact] = Field( + default=None, description="Emergency contact information on file" + ) + + +class HealthcareUserDB(DB): + """ + Database representing the patient's side of the interaction. + This contains information the patient can access but the agent cannot see directly. + """ + + patient_device: PatientDevice = Field( + description="Physical items and information patient can check" + ) + surroundings: PatientSurroundings = Field( + description="Context about the patient's current situation" + ) + + +# Export types for convenience +__all__ = [ + "InsuranceCard", + "Symptom", + "MedicationBottle", + "TimeSlot", + "PatientPortalInfo", + "BloodPressureReading", + "PainAssessment", + "EmergencyContact", + "PatientDevice", + "PatientSurroundings", + "HealthcareUserDB", + "SymptomSeverity", +] diff --git a/src/tau2/domains/healthcare/user_tools.py b/src/tau2/domains/healthcare/user_tools.py new file mode 100644 index 00000000..1d1bc811 --- /dev/null +++ b/src/tau2/domains/healthcare/user_tools.py @@ -0,0 +1,915 @@ +from typing import List, Optional + +from loguru import logger + +from tau2.domains.healthcare.user_data_model import ( + HealthcareUserDB, + InsuranceCard, + MedicationBottle, + PatientPortalInfo, + Symptom, + TimeSlot, +) +from tau2.environment.toolkit import ToolKitBase, ToolType, is_tool + + +class HealthcareUserTools(ToolKitBase): + """Patient-accessible tools for healthcare interactions.""" + + db: HealthcareUserDB + + def __init__(self, db: HealthcareUserDB) -> None: + super().__init__(db) + + @property + def device(self): + """Patient device.""" + return self.db.patient_device + + @property + def surroundings(self): + """Patient surroundings.""" + return self.db.surroundings + + @is_tool(ToolType.READ) + def check_insurance_card(self) -> str: + """ + Look at your insurance card and read the information printed on it. + This shows your insurance provider, policy number, group number, and copay information. + + Returns: + A formatted string with all information visible on the insurance card + """ + card = self.device.insurance_card + result = f"""Insurance Card Information: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Provider: {card.provider} +Member Name: {card.member_name} +Policy Number: {card.policy_number} +Group Number: {card.group_number} +Copay Info: {card.copay_info} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + logger.info( + f"Patient checked insurance card: {card.provider} - {card.policy_number}" + ) + return result + + @is_tool(ToolType.READ) + def check_symptoms(self) -> str: + """ + Describe the symptoms you are currently experiencing. + This tells you how you're feeling right now, including severity and duration. + + Returns: + Description of all current symptoms with severity and duration + """ + if not self.device.current_symptoms: + return "You are not experiencing any notable symptoms at the moment." + + result = "Current Symptoms:\n" + result += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + for i, symptom in enumerate(self.device.current_symptoms, 1): + result += f"{i}. {symptom.description}\n" + result += f" Severity: {symptom.severity.upper()}\n" + result += f" Duration: {symptom.duration}\n" + result += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + logger.info( + f"Patient checked symptoms: {len(self.device.current_symptoms)} symptoms" + ) + return result + + @is_tool(ToolType.READ) + def take_temperature(self) -> str: + """ + Use a thermometer to measure your current body temperature. + + Returns: + Your current temperature reading in Fahrenheit + """ + temp = self.device.current_temperature + + if temp is None: + return "You don't have a thermometer available to check your temperature." + + if temp < 97.0: + status = "below normal (hypothermia concern)" + elif temp < 99.0: + status = "normal" + elif temp < 100.4: + status = "slightly elevated" + elif temp < 103.0: + status = "fever" + else: + status = "high fever (seek immediate care)" + + result = f"Temperature Reading: {temp}°F ({status})" + logger.info(f"Patient took temperature: {temp}°F") + return result + + @is_tool(ToolType.READ) + def check_medication_bottle(self, medication_name: Optional[str] = None) -> str: + """ + Look at a medication bottle you have at home and read the label information. + If you have multiple medications, specify which one you want to check. + + Args: + medication_name: Optional name of specific medication to check + + Returns: + Information printed on the medication bottle label + """ + if not self.device.medications_at_home: + return "You don't have any medication bottles at home." + + if medication_name: + for med in self.device.medications_at_home: + if medication_name.lower() in med.medication_name.lower(): + return self._format_medication_bottle(med) + return ( + f"You don't have a medication bottle for '{medication_name}' at home." + ) + + if len(self.device.medications_at_home) == 1: + return self._format_medication_bottle(self.device.medications_at_home[0]) + + result = ( + f"You have {len(self.device.medications_at_home)} medication bottles:\n" + ) + for i, med in enumerate(self.device.medications_at_home, 1): + result += f"{i}. {med.medication_name} - {med.dosage}\n" + result += "\nSpecify which medication you want to check for full details." + return result + + def _format_medication_bottle(self, med: MedicationBottle) -> str: + """Format medication bottle information.""" + result = f"""Medication Bottle Label: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Prescription #: {med.prescription_number} +Medication: {med.medication_name} +Dosage: {med.dosage} +Refills Remaining: {med.refills_remaining} +Prescribing Doctor: {med.prescribing_doctor} +Pharmacy: {med.pharmacy_name} +Pharmacy Phone: {med.pharmacy_phone} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + logger.info(f"Patient checked medication: {med.medication_name}") + return result + + @is_tool(ToolType.READ) + def check_calendar(self, date: Optional[str] = None) -> str: + """ + Check your personal calendar to see your availability. + If a specific date is provided, shows availability for that day only. + + Args: + date: Optional specific date to check in YYYY-MM-DD format + + Returns: + Your calendar availability + """ + if not self.device.calendar_availability: + return "Your calendar is empty - you have no scheduled conflicts." + + if date: + slots = [s for s in self.device.calendar_availability if s.date == date] + if not slots: + return f"You have no conflicts on {date} - completely available." + + result = f"Your availability on {date}:\n" + result += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" + for slot in slots: + status = "✓ Available" if slot.available else f"✗ Busy - {slot.reason}" + result += f"{slot.time}: {status}\n" + return result + + result = "Your Calendar Availability:\n" + result += "━━━━━━━━━━━━━━━━��━━━━━━━━━━━━━━━━━━━━\n" + + dates = {} + for slot in self.device.calendar_availability: + if slot.date not in dates: + dates[slot.date] = [] + dates[slot.date].append(slot) + + for date, slots in sorted(dates.items()): + result += f"\n{date}:\n" + for slot in slots: + status = "✓ Available" if slot.available else f"✗ Busy - {slot.reason}" + result += f" {slot.time}: {status}\n" + + logger.info(f"Patient checked calendar: {len(dates)} dates") + return result + + @is_tool(ToolType.READ) + def open_patient_portal(self) -> str: + """ + Log in to your patient portal online to view your health information. + Shows upcoming appointments, recent visits, test results, messages, and billing. + + Returns: + Summary of information available in your patient portal + """ + if not self.surroundings.has_internet_access: + return ( + "You don't have internet access right now to open the patient portal." + ) + + portal = self.device.portal_info + if not portal: + return "Unable to access patient portal. You may need to contact the office for login credentials." + + result = """Patient Portal Dashboard: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +""" + + if portal.upcoming_appointments: + result += ( + f"\n📅 Upcoming Appointments ({len(portal.upcoming_appointments)}):\n" + ) + for apt in portal.upcoming_appointments: + result += f" • {apt}\n" + else: + result += "\n📅 Upcoming Appointments: None scheduled\n" + + if portal.recent_visits: + result += f"\n🏥 Recent Visits ({len(portal.recent_visits)}):\n" + for visit in portal.recent_visits: + result += f" • {visit}\n" + + if portal.test_results_available and portal.test_results: + result += f"\n🔬 Test Results ({len(portal.test_results)} available):\n" + for test in portal.test_results: + result += f" • {test['test_name']} ({test['test_date']})\n" + result += f" Result: {test['result']}\n" + if test.get("notes"): + result += f" Notes: {test['notes']}\n" + elif portal.test_results_available: + result += "\n🔬 Test Results: ✓ New results available to view\n" + else: + result += "\n🔬 Test Results: No new results\n" + + if portal.messages_count > 0: + result += f"\n✉️ Messages: {portal.messages_count} unread message(s)\n" + else: + result += "\n✉️ Messages: No new messages\n" + + if portal.outstanding_balance > 0: + result += f"\n💰 Outstanding Balance: ${portal.outstanding_balance}\n" + else: + result += "\n💰 Outstanding Balance: $0 - All paid\n" + + result += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + logger.info("Patient opened portal") + return result + + @is_tool(ToolType.READ) + def confirm_identity(self) -> str: + """ + Provide your identifying information for verification (name and date of birth). + This is used to confirm your identity before discussing health information. + + Returns: + Your full name and date of birth + """ + from datetime import datetime + + try: + dob = datetime.strptime(self.surroundings.date_of_birth, "%Y-%m-%d") + formatted_dob = dob.strftime("%B %d, %Y") + except (ValueError, TypeError): + formatted_dob = self.surroundings.date_of_birth + + result = f"""Identity Verification: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Full Name: {self.surroundings.full_name} +Date of Birth: {formatted_dob} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + + logger.info(f"Patient confirmed identity: {self.surroundings.patient_id}") + return result + + @is_tool(ToolType.WRITE) + def make_payment(self, amount: int, payment_method: str = "credit_card") -> str: + """ + Make a payment for medical services. + + Args: + amount: Amount to pay in dollars + payment_method: Payment method to use (credit_card, debit_card, cash) + + Returns: + Payment confirmation + """ + if payment_method not in self.surroundings.payment_methods_available: + available = ", ".join(self.surroundings.payment_methods_available) + return f"You don't have {payment_method} available. You can pay with: {available}" + + result = f"""Payment Confirmation: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Amount Paid: ${amount} +Payment Method: {payment_method.replace("_", " ").title()} +Status: ✓ APPROVED +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Thank you for your payment!""" + + logger.info(f"Patient made payment: ${amount} via {payment_method}") + return result + + @is_tool(ToolType.WRITE) + def confirm_appointment(self, appointment_id: str) -> str: + """ + Confirm that you will attend a scheduled appointment. + + Args: + appointment_id: The appointment ID to confirm + + Returns: + Confirmation message + """ + if appointment_id in self.device.confirmed_appointments: + return f"You have already confirmed appointment {appointment_id}." + + self.device.confirmed_appointments.append(appointment_id) + logger.info(f"Patient confirmed appointment: {appointment_id}") + + return f"""Appointment Confirmation: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Appointment ID: {appointment_id} +Status: ✓ CONFIRMED +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Thank you for confirming. We'll see you at your scheduled time!""" + + @is_tool(ToolType.WRITE) + def provide_consent(self, consent_type: str) -> str: + """ + Provide consent for treatment, procedures, or data sharing. + + Args: + consent_type: Type of consent (e.g., "telehealth", "treatment", "data_sharing", "billing") + + Returns: + Consent confirmation + """ + if consent_type in self.device.consents_provided: + return f"You have already provided consent for {consent_type}." + + self.device.consents_provided.append(consent_type) + logger.info(f"Patient provided consent: {consent_type}") + + return f"""Consent Provided: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Consent Type: {consent_type} +Status: ✓ AUTHORIZED +Date: {self._get_current_date()} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Your consent has been recorded and is now active.""" + + @is_tool(ToolType.WRITE) + def acknowledge_instructions(self, instruction_type: str) -> str: + """ + Acknowledge that you understand and will follow medical instructions. + + Args: + instruction_type: Type of instructions (e.g., "medication", "pre_surgery", "post_care", "diet") + + Returns: + Acknowledgment confirmation + """ + if instruction_type in self.device.acknowledged_instructions: + return f"You have already acknowledged {instruction_type} instructions." + + self.device.acknowledged_instructions.append(instruction_type) + logger.info(f"Patient acknowledged instructions: {instruction_type}") + + return f"""Instructions Acknowledged: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Type: {instruction_type} +Status: ✓ UNDERSTOOD +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +You have confirmed understanding of these instructions. +Please follow them as directed by your healthcare provider.""" + + @is_tool(ToolType.WRITE) + def update_emergency_contact(self, name: str, phone: str, relationship: str) -> str: + """ + Update your emergency contact information on file. + + Args: + name: Emergency contact's full name + phone: Emergency contact's phone number + relationship: Relationship to you (e.g., "spouse", "parent", "sibling", "friend") + + Returns: + Update confirmation + """ + from tau2.domains.healthcare.user_data_model import EmergencyContact + + self.surroundings.emergency_contact = EmergencyContact( + name=name, phone=phone, relationship=relationship + ) + logger.info(f"Patient updated emergency contact: {name} ({relationship})") + + return f"""Emergency Contact Updated: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Name: {name} +Phone: {phone} +Relationship: {relationship} +Status: ✓ UPDATED +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Your emergency contact information has been updated.""" + + @is_tool(ToolType.WRITE) + def enable_notification_preference(self, notification_type: str) -> str: + """ + Enable appointment reminders, test result alerts, or prescription refill reminders. + + Args: + notification_type: Type of notification ("appointment_reminders", "test_results", + "refill_reminders", "health_alerts") + + Returns: + Notification preference confirmation + """ + if notification_type in self.device.notification_preferences: + return f"{notification_type} notifications are already enabled." + + self.device.notification_preferences.append(notification_type) + logger.info(f"Patient enabled notification: {notification_type}") + + return f"""Notification Preference Updated: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Notification Type: {notification_type} +Status: ✓ ENABLED +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +You will now receive {notification_type} notifications.""" + + @is_tool(ToolType.WRITE) + def authorize_pharmacy_transfer( + self, medication_name: str, new_pharmacy: str + ) -> str: + """ + Authorize transferring a prescription to a different pharmacy. + + Args: + medication_name: Name of the medication to transfer + new_pharmacy: Name and location of the new pharmacy + + Returns: + Transfer authorization confirmation + """ + transfer_request = { + "medication_name": medication_name, + "new_pharmacy": new_pharmacy, + "requested_date": self._get_current_date(), + } + + self.device.pharmacy_transfer_requests.append(transfer_request) + logger.info( + f"Patient requested pharmacy transfer: {medication_name} to {new_pharmacy}" + ) + + return f"""Pharmacy Transfer Request: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Medication: {medication_name} +New Pharmacy: {new_pharmacy} +Status: ✓ AUTHORIZED +━━━━━��━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Your transfer request has been submitted. +The new pharmacy will contact your current pharmacy to complete the transfer. +This typically takes 1-2 business days.""" + + def _get_current_date(self) -> str: + """Helper to get current date for confirmations.""" + from datetime import datetime + + return datetime.now().strftime("%Y-%m-%d") + + # ============================================================================ + # INITIALIZATION ACTIONS - Used to set up test scenarios + # ============================================================================ + + def set_user_info(self, name: str, patient_id: str, date_of_birth: str) -> None: + """Set the patient's identifying information for scenario initialization.""" + self.surroundings.full_name = name + self.surroundings.patient_id = patient_id + self.surroundings.date_of_birth = date_of_birth + logger.info(f"Initialized user info: {name} ({patient_id})") + + def set_user_location(self, location: str) -> None: + """Set the patient's current location.""" + self.surroundings.location = location # type: ignore + logger.info(f"Set user location: {location}") + + def set_insurance_info( + self, + provider: str, + policy_number: str, + group_number: str, + member_name: str, + copay_info: str, + ) -> None: + """Initialize insurance card information.""" + self.device.insurance_card = InsuranceCard( + provider=provider, + policy_number=policy_number, + group_number=group_number, + member_name=member_name, + copay_info=copay_info, + ) + logger.info(f"Set insurance: {provider} - {policy_number}") + + def add_medication_at_home( + self, + prescription_number: str, + medication_name: str, + dosage: str, + refills_remaining: int, + prescribing_doctor: str, + pharmacy_name: str, + pharmacy_phone: str, + ) -> None: + """Add a medication bottle to patient's home.""" + med = MedicationBottle( + prescription_number=prescription_number, + medication_name=medication_name, + dosage=dosage, + refills_remaining=refills_remaining, + prescribing_doctor=prescribing_doctor, + pharmacy_name=pharmacy_name, + pharmacy_phone=pharmacy_phone, + ) + self.device.medications_at_home.append(med) + logger.info(f"Added medication: {medication_name}") + + def add_symptom(self, description: str, severity: str, duration: str) -> None: + """Add a symptom the patient is experiencing.""" + symptom = Symptom( + description=description, + severity=severity, + duration=duration, # type: ignore + ) + self.device.current_symptoms.append(symptom) + logger.info(f"Added symptom: {description} ({severity})") + + def set_temperature(self, temperature: float) -> None: + """Set the patient's current body temperature.""" + self.device.current_temperature = temperature + logger.info(f"Set temperature: {temperature}°F") + + def add_calendar_slot( + self, date: str, time: str, available: bool, reason: Optional[str] = None + ) -> None: + """Add a time slot to the patient's calendar.""" + slot = TimeSlot(date=date, time=time, available=available, reason=reason) + self.device.calendar_availability.append(slot) + logger.info( + f"Added calendar slot: {date} {time} - {'Available' if available else 'Busy'}" + ) + + def set_portal_info( + self, + upcoming_appointments: List[str], + recent_visits: List[str], + test_results_available: bool, + messages_count: int, + outstanding_balance: float, + test_results: Optional[List[dict]] = None, + ) -> None: + """Initialize patient portal information.""" + self.device.portal_info = PatientPortalInfo( + upcoming_appointments=upcoming_appointments, + recent_visits=recent_visits, + test_results_available=test_results_available, + test_results=test_results if test_results is not None else [], + messages_count=messages_count, + outstanding_balance=outstanding_balance, # type: ignore + ) + logger.info("Initialized patient portal info") + + def set_bp_monitor(self, has_monitor: bool, systolic: int, diastolic: int) -> None: + """Set blood pressure monitor and reading.""" + from tau2.domains.healthcare.user_data_model import BloodPressureReading + + self.device.has_blood_pressure_monitor = has_monitor + if has_monitor: + self.device.latest_bp_reading = BloodPressureReading( + systolic=systolic, diastolic=diastolic + ) + logger.info(f"Set BP monitor: {systolic}/{diastolic} mmHg") + + def set_glucose_monitor( + self, has_monitor: bool, glucose_reading: int, measurement_time: str + ) -> None: + """Set glucose meter and reading.""" + self.device.has_glucose_meter = has_monitor + if has_monitor: + self.device.latest_glucose_reading = glucose_reading + self.device.glucose_measurement_time = measurement_time + logger.info( + f"Set glucose monitor: {glucose_reading} mg/dL ({measurement_time})" + ) + + def set_pulse_oximeter(self, has_monitor: bool, spo2: int, heart_rate: int) -> None: + """Set pulse oximeter and reading.""" + self.device.has_pulse_oximeter = has_monitor + if has_monitor: + self.device.latest_spo2_reading = spo2 + self.device.latest_heart_rate = heart_rate + logger.info(f"Set pulse oximeter: SpO2 {spo2}%, HR {heart_rate} bpm") + + def set_emergency_contact(self, name: str, phone: str, relationship: str) -> None: + """Set emergency contact information.""" + from tau2.domains.healthcare.user_data_model import EmergencyContact + + self.surroundings.emergency_contact = EmergencyContact( + name=name, phone=phone, relationship=relationship + ) + logger.info(f"Set emergency contact: {name} ({relationship})") + + # ============================================================================ + # ENV_ASSERTION METHODS - Used for deterministic evaluation + # ============================================================================ + + def assert_has_calendar_availability(self, expected: bool) -> bool: + """Assert whether patient has any calendar availability configured.""" + has_availability = len(self.device.calendar_availability) > 0 + return has_availability == expected + + def assert_has_insurance_card(self, expected: bool) -> bool: + """Assert whether patient has insurance card information.""" + has_card = self.device.insurance_card is not None + return has_card == expected + + def assert_insurance_provider(self, expected_provider: str) -> bool: + """Assert the insurance provider name.""" + if not self.device.insurance_card: + return False + return self.device.insurance_card.provider == expected_provider + + def assert_has_symptoms(self, expected: bool) -> bool: + """Assert whether patient has current symptoms.""" + has_symptoms = len(self.device.current_symptoms) > 0 + return has_symptoms == expected + + def assert_temperature_reading(self, expected_temp: float) -> bool: + """Assert the patient's current temperature reading.""" + return self.device.current_temperature == expected_temp + + def assert_medication_count(self, expected_count: int) -> bool: + """Assert the number of medications at home.""" + return len(self.device.medications_at_home) == expected_count + + def assert_has_portal_access(self, expected: bool) -> bool: + """Assert whether patient has portal information available.""" + has_portal = self.device.portal_info is not None + return has_portal == expected + + def assert_consent_provided(self, consent_type: str) -> bool: + """Assert whether patient has provided a specific type of consent.""" + return consent_type in self.device.consents_provided + + def assert_instructions_acknowledged(self, instruction_type: str) -> bool: + """Assert whether patient has acknowledged a specific type of instructions.""" + return instruction_type in self.device.acknowledged_instructions + + def assert_emergency_contact_updated(self, name: str, relationship: str) -> bool: + """Assert whether emergency contact has been updated with specific details.""" + if not self.surroundings.emergency_contact: + return False + contact = self.surroundings.emergency_contact + return contact.name == name and contact.relationship == relationship + + # ============================================================================ + # PATIENT MEDICAL MONITORING TOOLS - Home health measurements + # ============================================================================ + + @is_tool(ToolType.READ) + def measure_blood_pressure(self) -> str: + """ + Use a home blood pressure monitor to measure blood pressure. + + Returns: + Blood pressure reading with systolic/diastolic values + """ + if ( + not hasattr(self.device, "has_blood_pressure_monitor") + or not self.device.has_blood_pressure_monitor + ): + return "You don't have a blood pressure monitor at home." + + # Get simulated reading from device + reading = self.device.latest_bp_reading + + if reading is None: + return "You haven't taken your blood pressure yet. Please take a measurement first." + + systolic = reading.systolic + diastolic = reading.diastolic + + # Interpret the reading + if systolic >= 180 or diastolic >= 120: + status = "⚠️ HYPERTENSIVE CRISIS - Seek immediate medical attention" + elif systolic >= 140 or diastolic >= 90: + status = "High (Stage 2 Hypertension)" + elif systolic >= 130 or diastolic >= 80: + status = "Elevated (Stage 1 Hypertension)" + else: + status = "Normal" + + result = f"""Blood Pressure Reading: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Systolic: {systolic} mmHg +Diastolic: {diastolic} mmHg +Result: {systolic}/{diastolic} mmHg +Status: {status} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + + logger.info(f"Patient measured BP: {systolic}/{diastolic}") + return result + + @is_tool(ToolType.READ) + def measure_blood_glucose(self) -> str: + """ + Use a glucometer to measure blood glucose level. + + Returns: + Blood glucose reading in mg/dL + """ + if ( + not hasattr(self.device, "has_glucose_meter") + or not self.device.has_glucose_meter + ): + return "You don't have a glucose meter at home." + + reading = self.device.latest_glucose_reading # mg/dL + + if reading is None: + return "You haven't measured your blood glucose yet. Please take a measurement first." + + # Interpret reading (simplified - depends on fasting or not) + if reading < 70: + status = "⚠️ LOW (Hypoglycemia) - Consume fast-acting carbs immediately" + elif reading <= 100: + status = "Normal (fasting)" + elif reading <= 125: + status = "Elevated (Prediabetes range)" + else: + status = "⚠️ HIGH (Diabetes range)" + + result = f"""Blood Glucose Reading: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Glucose: {reading} mg/dL +Status: {status} +Time: {self.device.glucose_measurement_time} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + + logger.info(f"Patient measured glucose: {reading} mg/dL") + return result + + @is_tool(ToolType.READ) + def measure_oxygen_saturation(self) -> str: + """ + Use a pulse oximeter to measure blood oxygen saturation (SpO2). + + Returns: + SpO2 percentage and heart rate + """ + if ( + not hasattr(self.device, "has_pulse_oximeter") + or not self.device.has_pulse_oximeter + ): + return "You don't have a pulse oximeter at home." + + spo2 = self.device.latest_spo2_reading + heart_rate = self.device.latest_heart_rate + + if spo2 is None or heart_rate is None: + return "You haven't measured your oxygen saturation yet. Please take a measurement first." + + # Interpret reading + if spo2 < 85: + status = "⚠️ SEVERE HYPOXEMIA - Seek immediate medical attention" + elif spo2 < 90: + status = "Moderate Hypoxemia - Contact doctor" + elif spo2 < 95: + status = "Mild Hypoxemia - Monitor closely" + else: + status = "Normal" + + result = f"""Pulse Oximeter Reading: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +SpO2: {spo2}% +Heart Rate: {heart_rate} bpm +Status: {status} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + + logger.info(f"Patient measured SpO2: {spo2}%, HR: {heart_rate}") + return result + + @is_tool(ToolType.READ) + def describe_pain(self) -> str: + """ + Describe current pain using standardized PQRST assessment. + + Returns: + Pain assessment with provocation, quality, radiation, severity, and timing + """ + if not hasattr(self.device, "current_pain") or not self.device.current_pain: + return "You are not experiencing any significant pain right now." + + pain = self.device.current_pain + + result = f"""Pain Assessment (PQRST): +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +P - Provocation: {pain.provocation} +Q - Quality: {pain.quality} +R - Radiation: {pain.radiation} +S - Severity: {pain.severity}/10 +T - Timing: {pain.timing} +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━""" + + logger.info(f"Patient describing pain: severity {pain.severity}/10") + return result + + @is_tool(ToolType.WRITE) + def upload_photo(self, body_part: str, description: str) -> str: + """ + Upload a photo of symptoms (rash, wound, swelling, etc.). + Used in telehealth for visual assessment. + + Args: + body_part: Location of symptom (e.g., "left arm", "abdomen", "face") + description: Brief description of what's shown + + Returns: + Confirmation of photo upload + """ + from datetime import datetime + + photo_id = f"PHOTO_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + # Store photo metadata + photo_record = { + "photo_id": photo_id, + "body_part": body_part, + "description": description, + "uploaded_at": datetime.now().isoformat(), + } + self.device.uploaded_photos.append(photo_record) + + logger.info(f"Patient uploaded photo: {body_part} - {description}") + + return f"""Photo Upload Confirmation: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Photo ID: {photo_id} +Body Part: {body_part} +Description: {description} +Status: ✓ Uploaded successfully +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +The doctor will review this photo and may request additional views if needed.""" + + @is_tool(ToolType.READ) + def check_symptom_severity(self) -> str: + """ + Assess the overall severity of current symptoms. + Helps determine if urgent care is needed. + + Returns: + Summary of symptoms with severity assessment + """ + if ( + not hasattr(self.device, "current_symptoms") + or not self.device.current_symptoms + ): + return "You are not experiencing any symptoms at this time." + + symptoms = self.device.current_symptoms + + # Count by severity + severe = [s for s in symptoms if s.severity == "severe"] + moderate = [s for s in symptoms if s.severity == "moderate"] + mild = [s for s in symptoms if s.severity == "mild"] + + result = f"""Symptom Summary: +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Total Symptoms: {len(symptoms)} +Severe: {len(severe)} +Moderate: {len(moderate)} +Mild: {len(mild)} + +Details: +""" + for symptom in symptoms: + result += f"- {symptom.description} ({symptom.severity}) - Duration: {symptom.duration}\n" + + result += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + if severe: + result += "\n⚠️ You have severe symptoms. Consider seeking immediate medical attention." + + logger.info( + f"Patient checked symptoms: {len(symptoms)} total, {len(severe)} severe" + ) + return result diff --git a/src/tau2/domains/healthcare/utils.py b/src/tau2/domains/healthcare/utils.py new file mode 100644 index 00000000..364e8d00 --- /dev/null +++ b/src/tau2/domains/healthcare/utils.py @@ -0,0 +1,17 @@ +# Copyright Sierra +from pathlib import Path + +HEALTHCARE_DOMAIN_PATH = Path(__file__).parent +HEALTHCARE_DATA_PATH = ( + HEALTHCARE_DOMAIN_PATH.parent.parent.parent.parent + / "data" + / "tau2" + / "domains" + / "healthcare" +) +HEALTHCARE_DB_PATH = HEALTHCARE_DATA_PATH / "db.json" +HEALTHCARE_USER_DB_PATH = HEALTHCARE_DATA_PATH / "user_db.json" +HEALTHCARE_POLICY_PATH = HEALTHCARE_DATA_PATH / "policy.md" +HEALTHCARE_TASK_SET_PATH = HEALTHCARE_DATA_PATH / "tasks.json" +HEALTHCARE_TASK_SET_FULL_PATH = HEALTHCARE_DATA_PATH / "tasks_full.json" +HEALTHCARE_TASK_SET_SMALL_PATH = HEALTHCARE_DATA_PATH / "tasks_small.json" diff --git a/src/tau2/registry.py b/src/tau2/registry.py index 87d6810b..45947b1e 100644 --- a/src/tau2/registry.py +++ b/src/tau2/registry.py @@ -23,6 +23,20 @@ from tau2.domains.retail.environment import ( get_tasks_split as retail_domain_get_tasks_split, ) +from tau2.domains.healthcare.environment import ( + get_environment as healthcare_domain_get_environment, +) +from tau2.domains.healthcare.environment import get_tasks as healthcare_domain_get_tasks +from tau2.domains.healthcare.environment import ( + get_tasks_full as healthcare_domain_get_tasks_full, +) +from tau2.domains.healthcare.environment import ( + get_tasks_small as healthcare_domain_get_tasks_small, +) +from tau2.domains.healthcare.environment import ( + get_tasks_split as healthcare_domain_get_tasks_split, +) + from tau2.domains.telecom.environment import ( get_environment_manual_policy as telecom_domain_get_environment_manual_policy, ) @@ -39,6 +53,7 @@ from tau2.domains.telecom.environment import ( get_tasks_split as telecom_domain_get_tasks_split, ) + from tau2.environment.environment import Environment from tau2.user.base import BaseUser from tau2.user.user_simulator import DummyUser, UserSimulator @@ -244,6 +259,15 @@ def get_info(self) -> RegistryInfo: get_task_splits=telecom_domain_get_tasks_split, ) + registry.register_domain(healthcare_domain_get_environment, "healthcare") + registry.register_tasks(healthcare_domain_get_tasks_full, "healthcare_full") + registry.register_tasks(healthcare_domain_get_tasks_small, "healthcare_small") + registry.register_tasks( + healthcare_domain_get_tasks, + "healthcare", + get_task_splits=healthcare_domain_get_tasks_split, + ) + logger.debug( f"Default components registered successfully. Registry info: {json.dumps(registry.get_info().model_dump(), indent=2)}" ) diff --git a/tests/test_domains/test_healthcare/test_tools_healthcare.py b/tests/test_domains/test_healthcare/test_tools_healthcare.py new file mode 100644 index 00000000..2c8813bc --- /dev/null +++ b/tests/test_domains/test_healthcare/test_tools_healthcare.py @@ -0,0 +1,428 @@ +"""Tests for the healthcare tools module.""" + +import unittest +from pathlib import Path + +from tau2.domains.healthcare.data_model import HealthcareDB +from tau2.domains.healthcare.tools import HealthcareTools + +# Path to the healthcare database file +HEALTHCARE_DB_PATH = ( + Path(__file__).parents[3] / "data" / "tau2" / "domains" / "healthcare" / "db.json" +) + + +class TestHealthcareTools(unittest.TestCase): + """Test cases for the healthcare tools module.""" + + def setUp(self): + """Set up test fixtures, if any.""" + # Load the healthcare database + self.db = HealthcareDB.load(str(HEALTHCARE_DB_PATH)) + # Create the healthcare tools instance + self.tools = HealthcareTools(self.db) + + def test_db_loaded(self): + """Test that the database is loaded correctly.""" + self.assertIsNotNone(self.db) + self.assertTrue(len(self.db.patients) > 0) + self.assertTrue(len(self.db.doctors) > 0) + self.assertTrue(len(self.db.appointments) > 0) + self.assertTrue(len(self.db.prescriptions) > 0) + + # ========================================================================= + # Patient Lookup Tests + # ========================================================================= + + def test_get_patient_details_success(self): + """Test getting patient details with valid name and DOB.""" + patient = self.tools.get_patient_details( + full_name="Sarah Johnson", date_of_birth="1985-03-15" + ) + self.assertIsNotNone(patient) + self.assertEqual(patient.patient_id, "patient_001") + self.assertEqual(patient.name.first_name, "Sarah") + self.assertEqual(patient.name.last_name, "Johnson") + + def test_get_patient_details_not_found(self): + """Test getting patient with non-existent name.""" + with self.assertRaises(ValueError) as context: + self.tools.get_patient_details( + full_name="Nonexistent Person", date_of_birth="1900-01-01" + ) + self.assertIn("no patient found", str(context.exception).lower()) + + def test_get_patient_details_wrong_dob(self): + """Test getting patient with correct name but wrong DOB.""" + with self.assertRaises(ValueError) as context: + self.tools.get_patient_details( + full_name="Sarah Johnson", date_of_birth="1990-01-01" + ) + self.assertIn("no patient found", str(context.exception).lower()) + + # ========================================================================= + # Doctor Lookup Tests + # ========================================================================= + + def test_list_available_doctors_all(self): + """Test listing all available doctors.""" + doctors = self.tools.list_available_doctors() + self.assertGreater(len(doctors), 0) + for doctor in doctors: + self.assertHasAttr(doctor, "doctor_id") + self.assertHasAttr(doctor, "name") + self.assertHasAttr(doctor, "specialty") + + def test_list_available_doctors_by_specialty(self): + """Test listing doctors filtered by specialty.""" + doctors = self.tools.list_available_doctors(specialty="General Practice") + self.assertGreater(len(doctors), 0) + for doctor in doctors: + self.assertEqual(doctor.specialty, "General Practice") + + def test_list_available_doctors_specialty_not_found(self): + """Test listing doctors with non-existent specialty.""" + doctors = self.tools.list_available_doctors(specialty="Nonexistent Specialty") + self.assertEqual(len(doctors), 0) + + # ========================================================================= + # Appointment Lookup Tests + # ========================================================================= + + def test_get_appointment_details_success(self): + """Test getting appointment details with valid ID.""" + # Use first appointment from DB + appointment_id = list(self.db.appointments.keys())[0] + appointment = self.tools.get_appointment_details(appointment_id) + self.assertIsNotNone(appointment) + self.assertEqual(appointment.appointment_id, appointment_id) + + def test_get_appointment_details_not_found(self): + """Test getting appointment with non-existent ID.""" + with self.assertRaises(ValueError) as context: + self.tools.get_appointment_details("nonexistent_appointment") + self.assertIn("not found", str(context.exception).lower()) + + def test_search_appointments_by_patient(self): + """Test searching appointments for a specific patient.""" + appointments = self.tools.search_appointments(patient_id="patient_001") + self.assertIsInstance(appointments, list) + for appt in appointments: + self.assertEqual(appt.patient_id, "patient_001") + + def test_search_appointments_by_status(self): + """Test searching appointments by status.""" + appointments = self.tools.search_appointments( + patient_id="patient_001", status="scheduled" + ) + self.assertIsInstance(appointments, list) + for appt in appointments: + self.assertEqual(appt.status, "scheduled") + + def test_search_appointments_patient_not_found(self): + """Test searching appointments for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.search_appointments(patient_id="nonexistent_patient") + + # ========================================================================= + # Appointment Time Slot Tests + # ========================================================================= + + def test_check_available_time_slots_success(self): + """Test checking available time slots for a doctor.""" + doctor_id = list(self.db.doctors.keys())[0] + slots = self.tools.check_available_time_slots( + doctor_id=doctor_id, date="2024-06-15" + ) + self.assertIsInstance(slots, list) + for slot in slots: + self.assertIn("time", slot) + self.assertIn("available", slot) + + def test_check_available_time_slots_doctor_not_found(self): + """Test checking time slots for non-existent doctor.""" + with self.assertRaises(ValueError): + self.tools.check_available_time_slots( + doctor_id="nonexistent_doctor", date="2024-06-15" + ) + + # ========================================================================= + # Appointment Booking Tests + # ========================================================================= + + def test_book_appointment_success(self): + """Test booking a new appointment.""" + result = self.tools.book_appointment( + patient_id="patient_001", + doctor_id="doc_001", + appointment_type="routine_checkup", + date="2024-12-02", + time="14:00", + reason="Annual checkup", + ) + self.assertIsNotNone(result) + self.assertEqual(result.status, "scheduled") + self.assertEqual(result.patient_id, "patient_001") + self.assertEqual(result.doctor_id, "doc_001") + self.assertIn(result.appointment_id, self.db.appointments) + + def test_book_appointment_patient_not_found(self): + """Test booking appointment for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.book_appointment( + patient_id="nonexistent_patient", + doctor_id="doc_001", + appointment_type="routine_checkup", + date="2024-12-02", + time="14:00", + reason="Test", + ) + + def test_book_appointment_doctor_not_found(self): + """Test booking appointment with non-existent doctor.""" + with self.assertRaises(ValueError): + self.tools.book_appointment( + patient_id="patient_001", + doctor_id="nonexistent_doctor", + appointment_type="routine_checkup", + date="2024-12-02", + time="14:00", + reason="Test", + ) + + # ========================================================================= + # Appointment Modification Tests + # ========================================================================= + + def test_cancel_appointment_success(self): + """Test canceling an existing appointment.""" + booking_result = self.tools.book_appointment( + patient_id="patient_001", + doctor_id="doc_001", + appointment_type="routine_checkup", + date="2024-12-02", + time="10:00", + reason="Test appointment for cancellation", + ) + appointment_id = booking_result.appointment_id + + result = self.tools.cancel_appointment( + appointment_id=appointment_id, reason="Patient canceled" + ) + self.assertEqual(result.status, "cancelled") + self.assertEqual(result.appointment_id, appointment_id) + + def test_cancel_appointment_not_found(self): + """Test canceling non-existent appointment.""" + with self.assertRaises(ValueError): + self.tools.cancel_appointment( + appointment_id="nonexistent_appointment", reason="Test" + ) + + def test_reschedule_appointment_success(self): + """Test rescheduling an existing appointment.""" + booking_result = self.tools.book_appointment( + patient_id="patient_001", + doctor_id="doc_001", + appointment_type="routine_checkup", + date="2024-12-03", + time="10:00", + reason="Test appointment for rescheduling", + ) + appointment_id = booking_result.appointment_id + + result = self.tools.reschedule_appointment( + appointment_id=appointment_id, new_date="2024-12-10", new_time="15:00" + ) + self.assertEqual(result.appointment_id, appointment_id) + self.assertEqual(result.date, "2024-12-10") + self.assertEqual(result.time, "15:00") + + def test_reschedule_appointment_not_found(self): + """Test rescheduling non-existent appointment.""" + with self.assertRaises(ValueError): + self.tools.reschedule_appointment( + appointment_id="nonexistent_appointment", + new_date="2024-12-10", + new_time="15:00", + ) + + # ========================================================================= + # Insurance Tests + # ========================================================================= + + def test_verify_insurance_coverage_success(self): + """Test verifying insurance coverage for a patient.""" + result = self.tools.verify_insurance_coverage( + patient_id="patient_001", procedure_type="routine_checkup" + ) + self.assertIn("verified", result) + self.assertIn("provider", result) + self.assertIn("policy_number", result) + self.assertIn("copay_amount", result) + self.assertIn("coverage_details", result) + self.assertIn("procedure_covered", result) + + def test_verify_insurance_coverage_patient_not_found(self): + """Test verifying insurance for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.verify_insurance_coverage( + patient_id="nonexistent_patient", procedure_type="routine_checkup" + ) + + # ========================================================================= + # Cost Calculation Tests + # ========================================================================= + + def test_calculate_cost_success(self): + """Test calculating cost for a procedure.""" + result = self.tools.calculate_cost( + appointment_type="routine_checkup", + insurance_provider="BlueCross", + ) + self.assertIn("base_cost", result) + self.assertIn("copay", result) + self.assertIn("insurance_covers", result) + self.assertIn("patient_pays", result) + self.assertIsInstance(result["base_cost"], (int, float)) + self.assertIsInstance(result["copay"], (int, float)) + self.assertIsInstance(result["insurance_covers"], (int, float)) + self.assertIsInstance(result["patient_pays"], (int, float)) + + # ========================================================================= + # Prescription Tests + # ========================================================================= + + def test_get_prescription_details_success(self): + """Test getting prescription details.""" + prescription_id = list(self.db.prescriptions.keys())[0] + prescription = self.tools.get_prescription_details(prescription_id) + self.assertIsNotNone(prescription) + self.assertEqual(prescription.prescription_id, prescription_id) + + def test_get_prescription_details_not_found(self): + """Test getting non-existent prescription.""" + with self.assertRaises(ValueError): + self.tools.get_prescription_details("nonexistent_prescription") + + def test_request_prescription_refill_success(self): + """Test requesting prescription refill.""" + prescription = next( + (p for p in self.db.prescriptions.values() if p.refills_remaining > 0), + None, + ) + self.assertIsNotNone(prescription, "No prescription with refills available") + + initial_refills = prescription.refills_remaining + result = self.tools.request_prescription_refill( + patient_id=prescription.patient_id, + prescription_id=prescription.prescription_id, + ) + self.assertEqual(result.prescription_id, prescription.prescription_id) + self.assertEqual(result.refills_remaining, initial_refills - 1) + + def test_request_prescription_refill_no_refills(self): + """Test requesting refill when no refills remaining.""" + prescription = next( + (p for p in self.db.prescriptions.values() if p.refills_remaining == 0), + None, + ) + self.assertIsNotNone(prescription, "No prescription with 0 refills available") + + with self.assertRaises(ValueError) as context: + self.tools.request_prescription_refill( + patient_id=prescription.patient_id, + prescription_id=prescription.prescription_id, + ) + self.assertIn("cannot refill prescription", str(context.exception).lower()) + + def test_request_prescription_refill_not_found(self): + """Test requesting refill for non-existent prescription.""" + with self.assertRaises(ValueError): + self.tools.request_prescription_refill( + patient_id="patient_001", prescription_id="nonexistent_prescription" + ) + + # ========================================================================= + # Test Results Tests + # ========================================================================= + + def test_check_test_results_success(self): + """Test checking test results for a patient.""" + results = self.tools.check_test_results(patient_id="patient_001") + self.assertIsInstance(results, list) + + def test_check_test_results_patient_not_found(self): + """Test checking test results for non-existent patient.""" + results = self.tools.check_test_results(patient_id="nonexistent_patient") + self.assertIsInstance(results, list) + self.assertEqual(len(results), 0) + + # ========================================================================= + # Chronic Condition Monitoring Tests + # ========================================================================= + + def test_get_vital_signs_history_success(self): + """Test getting vital signs history for a patient.""" + result = self.tools.get_vital_signs_history(patient_id="patient_001") + self.assertIsInstance(result, list) + for record in result: + self.assertIn("date", record) + self.assertIn("vital_type", record) + self.assertIn("value", record) + + def test_get_vital_signs_history_patient_not_found(self): + """Test getting vital signs for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.get_vital_signs_history(patient_id="nonexistent_patient") + + def test_get_chronic_conditions_success(self): + """Test getting chronic conditions for a patient.""" + result = self.tools.get_chronic_conditions(patient_id="patient_001") + self.assertIsInstance(result, list) + for condition in result: + self.assertIn("condition_name", condition) + self.assertIn("diagnosed_date", condition) + + def test_get_chronic_conditions_patient_not_found(self): + """Test getting chronic conditions for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.get_chronic_conditions(patient_id="nonexistent_patient") + + def test_get_lab_results_success(self): + """Test getting lab results for a patient.""" + result = self.tools.get_lab_results(patient_id="patient_001") + self.assertIsInstance(result, list) + + def test_get_lab_results_patient_not_found(self): + """Test getting lab results for non-existent patient.""" + with self.assertRaises(ValueError): + self.tools.get_lab_results(patient_id="nonexistent_patient") + + # ========================================================================= + # Transfer/Escalation Tests + # ========================================================================= + + def test_transfer_to_nurse(self): + """Test transferring to nurse.""" + result = self.tools.transfer_to_nurse() + self.assertIsInstance(result, str) + self.assertIn("transfer", result.lower()) + + def test_transfer_to_human_agent(self): + """Test transferring to human agent.""" + result = self.tools.transfer_to_human_agent() + self.assertIsInstance(result, str) + self.assertIn("transfer", result.lower()) + + # ========================================================================= + # Helper Methods + # ========================================================================= + + def assertHasAttr(self, obj, attr): + """Assert that object has attribute.""" + self.assertTrue(hasattr(obj, attr), f"Object does not have attribute '{attr}'") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_domains/test_healthcare/test_user_tools_healthcare.py b/tests/test_domains/test_healthcare/test_user_tools_healthcare.py new file mode 100644 index 00000000..6cf40a74 --- /dev/null +++ b/tests/test_domains/test_healthcare/test_user_tools_healthcare.py @@ -0,0 +1,290 @@ +"""Tests for the healthcare user tools module.""" + +import unittest + +from tau2.domains.healthcare.environment import get_environment + + +class TestHealthcareUserTools(unittest.TestCase): + """Test cases for the healthcare user tools module.""" + + def setUp(self): + """Set up test fixtures.""" + self.env = get_environment() + self.user_tools = self.env.user_tools + + # ========================================================================= + # Insurance and Identity Tests + # ========================================================================= + + def test_check_insurance_card(self): + """Test checking insurance card.""" + result = self.user_tools.check_insurance_card() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Provider", result) + self.assertIn("Policy Number", result) + + def test_confirm_identity(self): + """Test confirming patient identity.""" + result = self.user_tools.confirm_identity() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertTrue( + "patient" in result.lower() + or "name" in result.lower() + or "verification" in result.lower() + ) + + # ========================================================================= + # Symptom and Vital Signs Tests + # ========================================================================= + + def test_check_symptoms(self): + """Test checking symptoms.""" + result = self.user_tools.check_symptoms() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_take_temperature(self): + """Test taking temperature.""" + result = self.user_tools.take_temperature() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertTrue( + "°" in result + or "degrees" in result.lower() + or "temperature" in result.lower() + ) + + def test_describe_pain(self): + """Test describing pain.""" + result = self.user_tools.describe_pain() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_check_symptom_severity(self): + """Test checking symptom severity.""" + result = self.user_tools.check_symptom_severity() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + # ========================================================================= + # Medication Management Tests + # ========================================================================= + + def test_check_medication_bottle(self): + """Test checking medication bottle.""" + result = self.user_tools.check_medication_bottle() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_check_medication_bottle_specific(self): + """Test checking specific medication.""" + result = self.user_tools.check_medication_bottle(medication_name="Lisinopril") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_authorize_pharmacy_transfer(self): + """Test authorizing pharmacy transfer.""" + result = self.user_tools.authorize_pharmacy_transfer( + medication_name="Lisinopril", new_pharmacy="CVS Pharmacy - Main St" + ) + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Transfer", result) + self.assertIn("AUTHORIZED", result) + self.assertEqual(len(self.user_tools.device.pharmacy_transfer_requests), 1) + self.assertEqual( + self.user_tools.device.pharmacy_transfer_requests[0]["medication_name"], + "Lisinopril", + ) + self.assertEqual( + self.user_tools.device.pharmacy_transfer_requests[0]["new_pharmacy"], + "CVS Pharmacy - Main St", + ) + + # ========================================================================= + # Calendar and Scheduling Tests + # ========================================================================= + + def test_check_calendar(self): + """Test checking calendar.""" + result = self.user_tools.check_calendar() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_check_calendar_specific_date(self): + """Test checking calendar for specific date.""" + result = self.user_tools.check_calendar(date="2024-06-01") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + # ========================================================================= + # Patient Portal Tests + # ========================================================================= + + def test_open_patient_portal(self): + """Test opening patient portal.""" + result = self.user_tools.open_patient_portal() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + # ========================================================================= + # Home Monitoring Device Tests + # ========================================================================= + + def test_measure_blood_pressure(self): + """Test measuring blood pressure.""" + result = self.user_tools.measure_blood_pressure() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_measure_blood_glucose(self): + """Test measuring blood glucose.""" + result = self.user_tools.measure_blood_glucose() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + def test_measure_oxygen_saturation(self): + """Test measuring oxygen saturation.""" + result = self.user_tools.measure_oxygen_saturation() + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + + # ========================================================================= + # Consent and Confirmation Tests + # ========================================================================= + + def test_provide_consent(self): + """Test providing consent.""" + result = self.user_tools.provide_consent(consent_type="treatment") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Consent", result) + self.assertIn("AUTHORIZED", result) + self.assertIn("treatment", self.user_tools.device.consents_provided) + + def test_provide_consent_duplicate(self): + """Test providing same consent twice.""" + self.user_tools.provide_consent(consent_type="telehealth") + result = self.user_tools.provide_consent(consent_type="telehealth") + self.assertIn("already provided consent", result) + + def test_acknowledge_instructions(self): + """Test acknowledging instructions.""" + result = self.user_tools.acknowledge_instructions(instruction_type="medication") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Instructions Acknowledged", result) + self.assertIn("UNDERSTOOD", result) + self.assertIn("medication", self.user_tools.device.acknowledged_instructions) + + def test_acknowledge_instructions_duplicate(self): + """Test acknowledging same instructions twice.""" + self.user_tools.acknowledge_instructions(instruction_type="diet") + result = self.user_tools.acknowledge_instructions(instruction_type="diet") + self.assertIn("already acknowledged", result) + + def test_confirm_appointment(self): + """Test confirming appointment.""" + result = self.user_tools.confirm_appointment(appointment_id="appt_001") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Confirmation", result) + self.assertIn("CONFIRMED", result) + self.assertIn("appt_001", self.user_tools.device.confirmed_appointments) + + def test_confirm_appointment_duplicate(self): + """Test confirming same appointment twice.""" + self.user_tools.confirm_appointment(appointment_id="appt_002") + result = self.user_tools.confirm_appointment(appointment_id="appt_002") + self.assertIn("already confirmed", result) + + # ========================================================================= + # Payment Tests + # ========================================================================= + + def test_make_payment(self): + """Test making payment.""" + result = self.user_tools.make_payment(amount=50, payment_method="credit_card") + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Payment", result) + self.assertIn("50", result) + self.assertIn("APPROVED", result) + + def test_make_payment_unavailable_method(self): + """Test making payment with unavailable method.""" + result = self.user_tools.make_payment( + amount=25, payment_method="cryptocurrency" + ) + self.assertIn("don't have", result) + + # ========================================================================= + # Profile Update Tests + # ========================================================================= + + def test_update_emergency_contact(self): + """Test updating emergency contact.""" + result = self.user_tools.update_emergency_contact( + name="John Doe", phone="555-1234", relationship="spouse" + ) + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Emergency Contact Updated", result) + self.assertIn("John Doe", result) + self.assertIn("UPDATED", result) + self.assertEqual( + self.env.user_tools.surroundings.emergency_contact.name, "John Doe" + ) + self.assertEqual( + self.env.user_tools.surroundings.emergency_contact.phone, "555-1234" + ) + self.assertEqual( + self.env.user_tools.surroundings.emergency_contact.relationship, "spouse" + ) + + def test_enable_notification_preference(self): + """Test enabling notification preference.""" + result = self.user_tools.enable_notification_preference( + notification_type="email" + ) + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Notification", result) + self.assertIn("ENABLED", result) + self.assertIn("email", self.user_tools.device.notification_preferences) + + def test_enable_notification_preference_duplicate(self): + """Test enabling same notification preference twice.""" + self.user_tools.enable_notification_preference(notification_type="test_results") + result = self.user_tools.enable_notification_preference( + notification_type="test_results" + ) + self.assertIn("already enabled", result) + + # ========================================================================= + # Photo Upload Tests + # ========================================================================= + + def test_upload_photo(self): + """Test uploading a photo of symptoms.""" + result = self.user_tools.upload_photo( + body_part="left arm", description="Red rash spreading from wrist to elbow" + ) + self.assertIsNotNone(result) + self.assertIsInstance(result, str) + self.assertIn("Photo Upload", result) + self.assertIn("left arm", result) + self.assertIn("Uploaded successfully", result) + self.assertEqual(len(self.user_tools.device.uploaded_photos), 1) + photo = self.user_tools.device.uploaded_photos[0] + self.assertEqual(photo["body_part"], "left arm") + self.assertEqual(photo["description"], "Red rash spreading from wrist to elbow") + self.assertIn("photo_id", photo) + self.assertIn("uploaded_at", photo) + + +if __name__ == "__main__": + unittest.main()