{
 "about": "Language models driving a real car. Each attempt below is one model's try at a mini-cone course in an empty parking lot, at walking pace, with a human in the driver's seat ready to brake. The model saw the road through two cameras and drove with three tools (observe, set_motion, stop_now); every command, the reason the model gave for it, the car's telemetry and the video are here, aligned to the frame. People and license plates are blurred. The harness that recorded this is open source: https://github.com/aditya-ramabadran/drivingbench_harness_v1.",
 "collections": {
  "2026-09-17-cone-course-trial": {
   "title": "Cone course trial, Sep 17 2026",
   "about": "One trial per model, up to three in-context attempts each, medium reasoning. Course: a backwards-U parking lot marked with lime/orange mini-cones and tall orange cones, finishing in a blue mini-cone parking area. An attempt ends when the supervising driver brakes for a mistake or an imminent collision; the operator then sends a fixed reflection prompt and, for the next attempt, a fixed continuation prompt in the same chat. GPT-6 Astra completed the course on its second attempt; no other attempt finished.",
   "groups": {
    "gpt-6-astra": {
     "title": "GPT-6 Astra (Codex, medium reasoning)",
     "about": "One Codex thread, recorded in two transcript files; the second continues the same context. Attempt 1 failed at the first corner; attempt 2 completed the course, the only full success of the trial.",
     "result": "F S",
     "public": true,
     "parts": {
      "attempt-1--fail--24137e": {
       "duration_s": 86.95,
       "calls": 21,
       "frames": 26,
       "public": true
      },
      "attempt-2--success--5c4479": {
       "duration_s": 331.0,
       "calls": 58,
       "frames": 66,
       "public": true
      }
     }
    },
    "gpt-5.6-sol": {
     "title": "GPT-5.6 Sol (Codex, medium reasoning)",
     "about": "Three short attempts in one Codex thread, each stopped by the supervising driver within a minute of the start.",
     "result": "F F F",
     "public": true,
     "parts": {
      "attempt-1--fail--5ceee7": {
       "duration_s": 42.8,
       "calls": 7,
       "frames": 8,
       "public": true
      },
      "attempt-2--fail--627be4": {
       "duration_s": 46.7,
       "calls": 11,
       "frames": 14,
       "public": true
      },
      "attempt-3--fail--48ff3a": {
       "duration_s": 43.5,
       "calls": 7,
       "frames": 10,
       "public": true
      }
     }
    },
    "claude-fable-5.1": {
     "title": "Claude Fable 5.1 (Claude Code, medium reasoning)",
     "about": "Three attempts in one Claude Code session. The first two were stopped near the start; the third drove the long section of the course before the supervising driver braked.",
     "result": "F F F",
     "public": true,
     "parts": {
      "attempt-1--fail--18ade2": {
       "duration_s": 66.05,
       "calls": 6,
       "frames": 6,
       "public": true
      },
      "attempt-2--fail--926568": {
       "duration_s": 255.8,
       "calls": 9,
       "frames": 10,
       "public": true
      },
      "attempt-3--fail--ca2652": {
       "duration_s": 329.15,
       "calls": 17,
       "frames": 18,
       "public": true
      }
     }
    },
    "grok-4.6": {
     "title": "Grok 4.6 (Cursor, medium reasoning)",
     "about": "Three attempts in one Cursor chat, each stopped by the supervising driver near the start of the course. Cursor replaces the in-flight turn when the operator presses stop and then types, so each attempt was ended by typing \"Stop\" in the chat after braking rather than by interrupting the turn. Cursor's transcript records neither tool outputs nor timestamps: tool outputs are in each attempt's timeline.json, and the trace viewer places the chat turns between the exact tool-call times.",
     "result": "F F F",
     "public": true,
     "parts": {
      "attempt-1--fail--f1762a": {
       "duration_s": 52.45,
       "calls": 6,
       "frames": 8,
       "public": true
      },
      "attempt-2--fail--503d24": {
       "duration_s": 64.8,
       "calls": 7,
       "frames": 8,
       "public": true
      },
      "attempt-3--fail--667691": {
       "duration_s": 57.35,
       "calls": 7,
       "frames": 8,
       "public": true
      }
     }
    }
   }
  }
 }
}