Merge pull request #86 from sierra-research/leaderboard/add-gemini-3

Leaderboard/Add gemini 3

Author: benshi34

web/leaderboard/public/submissions/gemini-3-pro_google_2025-11-18/submission.json

@@ -0,0 +1,51 @@
+{
+  "model_name": "Gemini 3.0 Pro",
+  "model_organization": "Google",
+  "submitting_organization": "Sierra",
+  "submission_date": "2025-11-18",
+  "contact_info": {},
+  "is_new": true,
+  "trajectories_available": false,
+  "references": [
+    {
+      "title": "Gemini Eval Documentation",
+      "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_3_pro_model_evaluation.pdf",
+      "type": "documentation"
+    }
+  ],
+  "results": {
+    "retail": {
+      "pass_1": 73.0,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null,
+      "cost": null
+    },
+    "airline": {
+      "pass_1": 85.3,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null,
+      "cost": null
+    },
+    "telecom": {
+      "pass_1": 98.0,
+      "pass_2": null,
+      "pass_3": null,
+      "pass_4": null,
+      "cost": null
+    }
+  },
+  "methodology": {
+    "evaluation_date": "2025-11-18",
+    "tau2_bench_version": "v0.1.3",
+    "user_simulator": "Gemini-3.0-Pro",
+    "notes": "τ2-bench results for Gemini use standard sierra framework with a prompt adjustment to provide instructions relevant to each environment. The user model uses Gemini with a system instruction. ",
+    "verification": {
+      "modified_prompts": true,
+      "omitted_questions": false,
+      "details": "Prompt adjustments made to provide instructions relevant to each environment. User model uses Gemini with system instruction."
+    }
+  }
+}
+

web/leaderboard/public/submissions/manifest.json

@@ -11,7 +11,8 @@
     "o3_openai_2025-01-15",
     "o4-mini_openai_2024-06-20",
     "gpt-5_sierra_2025-08-09",
-    "qwen3-max_qwen_2024_09_23"
+    "qwen3-max_qwen_2024_09_23",
+    "gemini-3-pro_google_2025-11-18"
   ],
-  "last_updated": "2025-10-11T01:30:00Z"
+  "last_updated": "2025-11-18T00:00:00Z"
 }