{
  "name": "CaseVBench v2: published aggregate results",
  "version": "v2",
  "release_date": "2026-09-05",
  "source_url": "https://coxit.co/ai-drawing-benchmark/",
  "methodology_url": "https://coxit.co/blog/ai-architectural-drawings-benchmark-part-2/",
  "checked_date": "2026-09-15",
  "research_organization": "COXIT",
  "research_partner": "LVL10",
  "pages": 119,
  "annotated_objects": 1430,
  "projects": 9,
  "iou_threshold": 0.5,
  "score_unit": "F1 percent (0–100)",
  "overall_aggregation": "Pooled across all objects; not the mean of category scores",
  "evaluation_mode": "Direct full-page object detection with bounding boxes",
  "model_count": 11,
  "notes": [
    "Model names and rounded scores are reproduced as reported by COXIT. Exact API model identifiers are not provided in these exports.",
    "Aggregate results only; not the source drawings, annotations, predictions, or evaluation code.",
    "Five models were added in v2. COXIT reports that the original six were re-scored with unchanged results."
  ],
  "models": [
    {
      "name": "GPT-6 Astra",
      "provider": "OpenAI",
      "f1_percent": {
        "overall": 92,
        "cabinets": 82,
        "countertops": 84,
        "plans": 99,
        "elevations": 94,
        "callouts": 96
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.186,
        "median_seconds_per_page": 28.2,
        "mean_usd_per_page": 0.22,
        "mean_seconds_per_page": 39,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Gemini 3.8 Flash",
      "provider": "Google",
      "f1_percent": {
        "overall": 70,
        "cabinets": 59,
        "countertops": 20,
        "plans": 97,
        "elevations": 86,
        "callouts": 68
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.142,
        "median_seconds_per_page": 100.1,
        "mean_usd_per_page": 0.15,
        "mean_seconds_per_page": 104,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Claude Fable 5.1",
      "provider": "Anthropic",
      "f1_percent": {
        "overall": 70,
        "cabinets": 61,
        "countertops": 34,
        "plans": 98,
        "elevations": 87,
        "callouts": 64
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.148,
        "median_seconds_per_page": 22.3,
        "mean_usd_per_page": 0.164,
        "mean_seconds_per_page": 26,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "GPT-5.6 Sol",
      "provider": "OpenAI",
      "f1_percent": {
        "overall": 65,
        "cabinets": 53,
        "countertops": 24,
        "plans": 98,
        "elevations": 88,
        "callouts": 57
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.114,
        "median_seconds_per_page": 45.2,
        "mean_usd_per_page": 0.12,
        "mean_seconds_per_page": 49,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Qwen3.8-Max",
      "provider": "Alibaba",
      "f1_percent": {
        "overall": 62,
        "cabinets": 60,
        "countertops": 32,
        "plans": 98,
        "elevations": 90,
        "callouts": 47
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.055,
        "median_seconds_per_page": 154.9,
        "mean_usd_per_page": 0.057,
        "mean_seconds_per_page": 161,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "GPT-5.6 Terra",
      "provider": "OpenAI",
      "f1_percent": {
        "overall": 56,
        "cabinets": 51,
        "countertops": 23,
        "plans": 94,
        "elevations": 87,
        "callouts": 38
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.056,
        "median_seconds_per_page": 32,
        "mean_usd_per_page": 0.056,
        "mean_seconds_per_page": 32,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Gemini 3.5 Flash",
      "provider": "Google",
      "f1_percent": {
        "overall": 53,
        "cabinets": 45,
        "countertops": 16,
        "plans": 96,
        "elevations": 90,
        "callouts": 33
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.026,
        "median_seconds_per_page": 15.5,
        "mean_usd_per_page": 0.035,
        "mean_seconds_per_page": 20,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Gemini 3.1 Pro Preview",
      "provider": "Google",
      "f1_percent": {
        "overall": 51,
        "cabinets": 39,
        "countertops": 10,
        "plans": 89,
        "elevations": 78,
        "callouts": 37
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.033,
        "median_seconds_per_page": 19.2,
        "mean_usd_per_page": 0.04,
        "mean_seconds_per_page": 23,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Claude Opus 5",
      "provider": "Anthropic",
      "f1_percent": {
        "overall": 40,
        "cabinets": 51,
        "countertops": 19,
        "plans": 96,
        "elevations": 89,
        "callouts": 15
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.112,
        "median_seconds_per_page": 37.2,
        "mean_usd_per_page": 0.134,
        "mean_seconds_per_page": 48,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Grok 4.6",
      "provider": "xAI",
      "f1_percent": {
        "overall": 30,
        "cabinets": 5,
        "countertops": 7,
        "plans": 91,
        "elevations": 69,
        "callouts": 0
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.134,
        "median_seconds_per_page": 331.9,
        "mean_usd_per_page": 0.145,
        "mean_seconds_per_page": 362,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    },
    {
      "name": "Claude Sonnet 5",
      "provider": "Anthropic",
      "f1_percent": {
        "overall": 21,
        "cabinets": 10,
        "countertops": 0,
        "plans": 73,
        "elevations": 47,
        "callouts": 4
      },
      "cost_and_time": {
        "measurement_period": "2026-09",
        "pages": 119,
        "median_usd_per_page": 0.072,
        "median_seconds_per_page": 60.8,
        "mean_usd_per_page": 0.073,
        "mean_seconds_per_page": 60,
        "median_source": "https://coxit.co/ai-drawing-benchmark/",
        "mean_source": "https://coxit.co/blog/gpt-6-astra-architectural-drawings-benchmark/"
      }
    }
  ]
}