{
  "name": "CaseVBench v1: published aggregate results",
  "version": "v1",
  "release_date": "2026-08-26",
  "source_url": "https://coxit.co/ai-drawing-benchmark/",
  "methodology_url": "https://coxit.co/blog/ai-architectural-drawings-benchmark-part-2/",
  "checked_date": "2026-09-15",
  "research_organization": "COXIT",
  "research_partner": "LVL10",
  "pages": 119,
  "annotated_objects": 1430,
  "projects": 9,
  "iou_threshold": 0.5,
  "score_unit": "F1 percent (0–100)",
  "overall_aggregation": "Pooled across all objects; not the mean of category scores",
  "evaluation_mode": "Direct full-page object detection with bounding boxes",
  "model_count": 6,
  "notes": [
    "Model names and rounded scores are reproduced as reported by COXIT. Exact API model identifiers are not provided in these exports.",
    "Aggregate results only; not the source drawings, annotations, predictions, or evaluation code."
  ],
  "models": [
    {
      "name": "Qwen3.8-Max",
      "provider": "Alibaba",
      "f1_percent": {
        "overall": 62,
        "cabinets": 60,
        "countertops": 32,
        "plans": 98,
        "elevations": 90,
        "callouts": 47
      }
    },
    {
      "name": "Gemini 3.5 Flash",
      "provider": "Google",
      "f1_percent": {
        "overall": 53,
        "cabinets": 45,
        "countertops": 16,
        "plans": 96,
        "elevations": 90,
        "callouts": 33
      }
    },
    {
      "name": "Gemini 3.1 Pro Preview",
      "provider": "Google",
      "f1_percent": {
        "overall": 51,
        "cabinets": 39,
        "countertops": 10,
        "plans": 89,
        "elevations": 78,
        "callouts": 37
      }
    },
    {
      "name": "Claude Opus 5",
      "provider": "Anthropic",
      "f1_percent": {
        "overall": 40,
        "cabinets": 51,
        "countertops": 19,
        "plans": 96,
        "elevations": 89,
        "callouts": 15
      }
    },
    {
      "name": "Grok 4.6",
      "provider": "xAI",
      "f1_percent": {
        "overall": 30,
        "cabinets": 5,
        "countertops": 7,
        "plans": 91,
        "elevations": 69,
        "callouts": 0
      }
    },
    {
      "name": "Claude Sonnet 5",
      "provider": "Anthropic",
      "f1_percent": {
        "overall": 21,
        "cabinets": 10,
        "countertops": 0,
        "plans": 73,
        "elevations": 47,
        "callouts": 4
      }
    }
  ]
}