{
  "name": "CaseVBench v4: published aggregate results",
  "version": "v4",
  "release_date": "2026-09-30",
  "source_url": "https://casevbench.com/#dataset-v4",
  "methodology_url": "https://coxit.co/blog/ai-architectural-drawings-benchmark-part-2/",
  "checked_date": "2026-09-30",
  "research_organization": "COXIT",
  "research_partner": "LVL10",
  "pages": 119,
  "annotated_objects": 1430,
  "projects": 9,
  "iou_threshold": 0.5,
  "score_unit": "F1 percent (0–100)",
  "overall_aggregation": "Pooled across all objects; not the mean of category scores",
  "evaluation_mode": "Direct full-page object detection with bounding boxes",
  "model_count": 18,
  "notes": [
    "F1 percentages computed from COXIT’s September 30 service export at IoU 0.50, pooled across projects and rounded to the nearest integer. Sample models are excluded. Claude Fable 5.1 is included in this public edition.",
    "Aggregate results only; not the source drawings, annotations, predictions, or evaluation code."
  ],
  "models": [
    {
      "name": "GPT-6 Astra",
      "provider": "OpenAI",
      "api_model_id": "openai/gpt-6-astra",
      "f1_percent": {
        "overall": 92,
        "cabinets": 82,
        "countertops": 84,
        "plans": 99,
        "elevations": 94,
        "callouts": 96
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.185705,
        "median_seconds_per_page": 28.228,
        "mean_usd_per_page": 0.2202599369747899,
        "mean_seconds_per_page": 39.27386554621849,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Claude Opus 5.5",
      "provider": "Anthropic",
      "api_model_id": "claude-opus-5.5",
      "f1_percent": {
        "overall": 79,
        "cabinets": 60,
        "countertops": 51,
        "plans": 91,
        "elevations": 90,
        "callouts": 84
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.04022,
        "median_seconds_per_page": 8.85482833699996,
        "mean_usd_per_page": 0.04520642016806723,
        "mean_seconds_per_page": 11.317569166563022,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Gemini 3.8 Flash",
      "provider": "Google",
      "api_model_id": "google/gemini-3.8-flash",
      "f1_percent": {
        "overall": 70,
        "cabinets": 59,
        "countertops": 20,
        "plans": 97,
        "elevations": 86,
        "callouts": 68
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.07146525,
        "median_seconds_per_page": 100.093,
        "mean_usd_per_page": 0.07412713235294116,
        "mean_seconds_per_page": 104.28503361344538,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4",
        "cost_note": "price reflects a 50% promotional discount, not standard rates"
      }
    },
    {
      "name": "Claude Fable 5.1",
      "provider": "Anthropic",
      "api_model_id": "anthropic/claude-fable-5.1",
      "f1_percent": {
        "overall": 70,
        "cabinets": 61,
        "countertops": 34,
        "plans": 98,
        "elevations": 87,
        "callouts": 64
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.14826,
        "median_seconds_per_page": 22.289,
        "mean_usd_per_page": 0.16358109243697477,
        "mean_seconds_per_page": 25.584857142857143,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "GPT-5.6 Sol",
      "provider": "OpenAI",
      "api_model_id": "openai/gpt-5.6-sol",
      "f1_percent": {
        "overall": 65,
        "cabinets": 53,
        "countertops": 24,
        "plans": 98,
        "elevations": 88,
        "callouts": 57
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.057156,
        "median_seconds_per_page": 45.192,
        "mean_usd_per_page": 0.060107113445378155,
        "mean_seconds_per_page": 48.69626050420168,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4",
        "cost_note": "price reflects a 50% promotional discount, not standard rates"
      }
    },
    {
      "name": "Qwen3.8-Max",
      "provider": "Alibaba",
      "api_model_id": "qwen/qwen3.8-max",
      "f1_percent": {
        "overall": 62,
        "cabinets": 60,
        "countertops": 32,
        "plans": 98,
        "elevations": 90,
        "callouts": 47
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.054924,
        "median_seconds_per_page": 154.875,
        "mean_usd_per_page": 0.05698712605042016,
        "mean_seconds_per_page": 161.21757983193277,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "GPT-6.1 Sol",
      "provider": "OpenAI",
      "api_model_id": "openai/gpt-6.1-sol",
      "f1_percent": {
        "overall": 60,
        "cabinets": 57,
        "countertops": 48,
        "plans": 95,
        "elevations": 65,
        "callouts": 51
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.061336,
        "median_seconds_per_page": 25.795898385999976,
        "mean_usd_per_page": 0.05526446050420168,
        "mean_seconds_per_page": 30.57984197957984,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "GPT-5.6 Terra",
      "provider": "OpenAI",
      "api_model_id": "openai/gpt-5.6-terra",
      "f1_percent": {
        "overall": 56,
        "cabinets": 51,
        "countertops": 23,
        "plans": 94,
        "elevations": 87,
        "callouts": 38
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.0558605,
        "median_seconds_per_page": 32.023,
        "mean_usd_per_page": 0.05598834033613445,
        "mean_seconds_per_page": 32.46021008403361,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Gemini 3.5 Flash",
      "provider": "Google",
      "api_model_id": "google/gemini-3.5-flash",
      "f1_percent": {
        "overall": 53,
        "cabinets": 45,
        "countertops": 16,
        "plans": 96,
        "elevations": 90,
        "callouts": 33
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.0264735,
        "median_seconds_per_page": 15.491,
        "mean_usd_per_page": 0.034911731092436976,
        "mean_seconds_per_page": 19.826201680672266,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Muse Spark 1.3",
      "provider": "Meta",
      "api_model_id": "muse-spark-1.3",
      "f1_percent": {
        "overall": 52,
        "cabinets": 52,
        "countertops": 21,
        "plans": 98,
        "elevations": 90,
        "callouts": 23
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.0198255,
        "median_seconds_per_page": 38.376088643,
        "mean_usd_per_page": 0.021630646218487395,
        "mean_seconds_per_page": 40.73714780542857,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Gemini 3.1 Pro Preview",
      "provider": "Google",
      "api_model_id": "google/gemini-3.1-pro-preview",
      "f1_percent": {
        "overall": 51,
        "cabinets": 39,
        "countertops": 10,
        "plans": 89,
        "elevations": 78,
        "callouts": 37
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.033339999999999995,
        "median_seconds_per_page": 19.247,
        "mean_usd_per_page": 0.03992636974789916,
        "mean_seconds_per_page": 23.04433613445378,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Claude Sonnet 5.5",
      "provider": "Anthropic",
      "api_model_id": "anthropic/claude-sonnet-5.5",
      "f1_percent": {
        "overall": 51,
        "cabinets": 48,
        "countertops": 34,
        "plans": 96,
        "elevations": 80,
        "callouts": 30
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.01968,
        "median_seconds_per_page": 5.545538919999899,
        "mean_usd_per_page": 0.020498420168067227,
        "mean_seconds_per_page": 6.140479917369765,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "GPT-6 Sol",
      "provider": "OpenAI",
      "api_model_id": "gpt-6-sol",
      "f1_percent": {
        "overall": 49,
        "cabinets": 25,
        "countertops": 11,
        "plans": 96,
        "elevations": 83,
        "callouts": 36
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.067976,
        "median_seconds_per_page": 24.253779301999888,
        "mean_usd_per_page": 0.06450070588235293,
        "mean_seconds_per_page": 25.636458104462186,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Claude Opus 5",
      "provider": "Anthropic",
      "api_model_id": "anthropic/claude-opus-5",
      "f1_percent": {
        "overall": 40,
        "cabinets": 51,
        "countertops": 19,
        "plans": 96,
        "elevations": 89,
        "callouts": 15
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.111555,
        "median_seconds_per_page": 37.211,
        "mean_usd_per_page": 0.1341687394957983,
        "mean_seconds_per_page": 48.30341176470588,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Grok 4.6",
      "provider": "xAI",
      "api_model_id": "x-ai/grok-4.6",
      "f1_percent": {
        "overall": 30,
        "cabinets": 5,
        "countertops": 7,
        "plans": 91,
        "elevations": 69,
        "callouts": 0
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.134276,
        "median_seconds_per_page": 331.912,
        "mean_usd_per_page": 0.1448843025210084,
        "mean_seconds_per_page": 362.32667226890754,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "GPT-6 Luna",
      "provider": "OpenAI",
      "api_model_id": "gpt-6-luna",
      "f1_percent": {
        "overall": 23,
        "cabinets": 13,
        "countertops": 8,
        "plans": 82,
        "elevations": 38,
        "callouts": 6
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.0038888,
        "median_seconds_per_page": 29.522218346999804,
        "mean_usd_per_page": 0.0036150100840336135,
        "mean_seconds_per_page": 29.36550888458824,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Claude Sonnet 5",
      "provider": "Anthropic",
      "api_model_id": "anthropic/claude-sonnet-5",
      "f1_percent": {
        "overall": 21,
        "cabinets": 10,
        "countertops": 0,
        "plans": 73,
        "elevations": 47,
        "callouts": 4
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.072366,
        "median_seconds_per_page": 60.773,
        "mean_usd_per_page": 0.07270774789915967,
        "mean_seconds_per_page": 60.24371428571428,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    },
    {
      "name": "Grok 4.7",
      "provider": "xAI",
      "api_model_id": "grok-4.7",
      "f1_percent": {
        "overall": 20,
        "cabinets": 3,
        "countertops": 0,
        "plans": 81,
        "elevations": 44,
        "callouts": 0
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.0778112,
        "median_seconds_per_page": 185.9361556670001,
        "mean_usd_per_page": 0.08546948571428571,
        "mean_seconds_per_page": 215.8832101412185,
        "median_source": "https://casevbench.com/#dataset-v4",
        "mean_source": "https://casevbench.com/#dataset-v4"
      }
    }
  ]
}