Submit Toolorchestra to leaderboard - Revised (#119)

* upload

* update

* Update submission.json

* toolorchestra submission + custom verification logic

* address comments

---------

Co-authored-by: hongjin-su <114016954+hongjin-su@users.noreply.github.com>

Author: benshi34

web/leaderboard/public/Logo-nvidia-transparent-PNG.png

@@ -14,7 +14,8 @@
     "qwen3-max_qwen_2024_09_23",
     "gemini-3-pro_google_2025-11-18",
     "deepseek-v3.2_deepseek_2025-12-01",
-    "qwen3-max_qwen_2025-10-30"
+    "qwen3-max_qwen_2025-10-30",
+    "toolorchestra_nvidia_2025-12-02"
   ],
-  "last_updated": "2025-12-01T00:00:00Z"
+  "last_updated": "2025-12-17T00:00:00Z"
 }

web/leaderboard/public/submissions/manifest.json

@@ -0,0 +1,56 @@
+{
+  "model_name": "Nemotron-Orchestrator-8B",
+  "model_organization": "NVIDIA",
+  "submitting_organization": "NVIDIA",
+  "submission_date": "2025-12-18",
+  "contact_info": {
+    "email": "sdiao@nvidia.com",
+    "name": "Shizhe Diao"
+  },
+  "is_new": true,
+  "trajectories_available": true,
+  "results": {
+    "retail": {
+      "pass_1": 84.2,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null
+    },
+    "airline": {
+      "pass_1": 56.0,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null
+    },
+    "telecom": {
+      "pass_1": 88.6,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null
+    }
+  },
+  "submission_type": "custom",
+  "methodology": {
+    "evaluation_date": "2025-09-15",
+    "tau2_bench_version": "0.1.3",
+    "user_simulator": "gpt-5",
+    "notes": "This submission uses an orchestrator that selects between strong and weak models based on task complexity. See our GitHub repo for full implementation details.",
+    "verification": {
+      "modified_prompts": true,
+      "omitted_questions": false,
+      "details": "Modified the agent system prompt to support orchestrator framework. No questions were omitted."
+    }
+  },
+  "references": [
+    {
+      "title": "ToolOrchestra: Elevating Intelligence via Efficient Model and Tool Orchestration",
+      "url": "https://arxiv.org/abs/2511.21689",
+      "type": "paper"
+    },
+    {
+      "title": "ToolOrchestra GitHub Repository",
+      "url": "https://github.com/NVlabs/ToolOrchestra",
+      "type": "github"
+    }
+  ]
+}

web/leaderboard/public/submissions/toolorchestra_nvidia_2025-12-02/submission.json

@@ -143,9 +143,11 @@ const Leaderboard = () => {
             organization: submission.submitting_organization,
             userSimulator: submission.methodology?.user_simulator || null,
             // Add verification status
+            // For 'custom' submissions, we relax the modified_prompts constraint
+            // Custom submissions are allowed to modify prompts as long as they have trajectories and don't omit questions
             isVerified: submission.trajectories_available && 
-                       submission.methodology?.verification?.modified_prompts === false && 
-                       submission.methodology?.verification?.omitted_questions === false,
+                       submission.methodology?.verification?.omitted_questions === false &&
+                       (submission.submission_type === 'custom' || submission.methodology?.verification?.modified_prompts === false),
             verificationDetails: submission.methodology?.verification || null,
             // Submission type: 'standard' (default) or 'custom'
             submissionType: submission.submission_type || 'standard'
@@ -800,10 +802,13 @@ const Leaderboard = () => {
                           {(model.organization === 'Alibaba' || model.organization === 'Qwen') && (
                             <img src={`${import.meta.env.BASE_URL}qwen-color.png`} alt="Qwen" className="logo-img" />
                           )}
-                          {model.organization === 'Google' && (
-                            <img src={`${import.meta.env.BASE_URL}Google__G__logo.svg.png`} alt="Google" className="logo-img" />
-                          )}
-                         </div>
+                         {model.organization === 'Google' && (
+                           <img src={`${import.meta.env.BASE_URL}Google__G__logo.svg.png`} alt="Google" className="logo-img" />
+                         )}
+                         {model.organization === 'NVIDIA' && (
+                           <img src={`${import.meta.env.BASE_URL}Logo-nvidia-transparent-PNG.png`} alt="NVIDIA" className="logo-img" />
+                         )}
+                        </div>
                          <span className="org-name">{model.organization}</span>
                        </div>
                      </td>
@@ -1076,8 +1081,8 @@ const Leaderboard = () => {
                     <div className="verification-status">
                       <div className="verification-indicator">
                         {selectedSubmission.trajectories_available && 
-                         selectedSubmission.methodology.verification.modified_prompts === false && 
-                         selectedSubmission.methodology.verification.omitted_questions === false ? (
+                         selectedSubmission.methodology.verification.omitted_questions === false &&
+                         (selectedSubmission.submission_type === 'custom' || selectedSubmission.methodology.verification.modified_prompts === false) ? (
                           <span className="verified">✅ Verified</span>
                         ) : (
                           <span className="unverified">⚠️ Unverified</span>

web/leaderboard/public/submissions/toolorchestra_nvidia_2025-12-02/trajectories/toolorchestra_airline_gpt-5_1trial.json

@@ -140,6 +140,9 @@ const TrajectoryVisualizer = () => {
         ],
         'Qwen3-Max-Thinking-Preview': [
           '{domain}_llm_agent_qwen3-max-2025-10-30_user_simulator_gpt-4.1-2025-04-14.json'
+        ],
+        'ToolOrchestra': [
+          'toolorchestra_{domain}_gpt-5_1trial.json'
         ]
       }