2026-10-22-claim-ledger
{
"schema_version": "1.0",
"research_date": "2026-10-22",
"scope": "Public unit-price and inference-work evidence for Knauf Digital presentation; no claims about Knauf measured spending",
"sources_file": "2026-10-22-source-manifest.json",
"claims": [
{
"id": "UP01",
"claim": "A non-reasoning model frontier meeting MMLU >= 64.8 fell from USD 20 to USD 0.07 per million blended API tokens from November 2022 to October 2024.",
"verdict": "supported_under_checked_conditions",
"priority": "main_chart",
"source_ids": [
"UP-S01",
"UP-S02"
],
"metric": "USD per 1M API tokens weighted 3:1 input/output",
"threshold": {
"benchmark": "MMLU",
"minimum_percent": 64.8
},
"endpoints": [
{
"date": "2022-11-30",
"model": "GPT-3.5",
"usd_per_million_tokens": 20,
"benchmark_score_percent": 64.8
},
{
"date": "2024-10-03",
"model": "Gemini-1.5-Flash-8B",
"usd_per_million_tokens": 0.07,
"benchmark_score_percent": 75
}
],
"derived": {
"ratio_from_rounded_values": 285.71428571428567,
"display_ratio": "more than 280x",
"elapsed_days": 673,
"method": "rounded endpoint division and Gregorian calendar subtraction"
},
"caveats": [
"Changing-model frontier, not same-model price cut",
"Benchmark threshold is not general capability equivalence",
"Not current October 2026 prices",
"Not task cost; no enterprise review/tools/infrastructure included",
"Reasoning models excluded from upstream analysis"
],
"chart_data": "2026-10-22-fixed-threshold-price-chart.csv"
},
{
"id": "UP02",
"claim": "The per-token price study excluded reasoning models because their longer generated traces can make token-price comparisons misleading for total evaluation cost.",
"verdict": "supported_under_checked_conditions",
"priority": "spoken_bridge",
"source_ids": [
"UP-S02"
],
"locator": "Data methodology paragraph on reasoning models",
"interpretation": "Measure tokens and complete workflow consumption per accepted result rather than inferring task savings from a token-price frontier."
},
{
"id": "UP03",
"claim": "In OpenAI's historical o1 AIME 2024 experiment, pass@1 was 74.4% and consensus over 64 samples was 83.3%.",
"verdict": "supported_as_vendor_reported_experiment",
"priority": "optional_main_chart",
"source_ids": [
"UP-S04"
],
"denominator": "AIME 2024 problem accuracy; article expresses mean against 15 questions per exam",
"configurations": [
{
"candidates_per_problem": 1,
"selection": "single_sample",
"accuracy_percent": 74.4
},
{
"candidates_per_problem": 64,
"selection": "consensus",
"accuracy_percent": 83.3
}
],
"derived": {
"accuracy_gain_percentage_points": 8.9,
"method": "83.3-74.4"
},
"caveats": [
"Historical research o1, not o1-preview/current models",
"No task-dollar or exact FLOPs ratio follows from candidate count",
"Per-sample reasoning length and selection overhead are not disclosed",
"Competition mathematics does not establish product-data workflow benefit",
"Vendor-reported; no independently reproduced outcome in this audit"
],
"chart_data": "2026-10-22-o1-sampling-chart.csv"
},
{
"id": "UP03-BACKUP",
"claim": "The same OpenAI article reports rounded 93% AIME accuracy after learned reranking of 1000 candidates.",
"verdict": "supported_as_vendor_reported_experiment",
"priority": "backup_only",
"source_ids": [
"UP-S04"
],
"caveats": [
"Main-text rounded value rather than appendix precision",
"Selection method changes in addition to sample count",
"Do not label 1000 candidate solutions as 1000 times an API bill"
],
"chart_data": "2026-10-22-o1-sampling-chart.csv"
},
{
"id": "UP04",
"claim": "Snell et al. report adaptive search/revision comparisons using about one quarter of a best-of-N baseline generation budget at similar or better MATH performance.",
"verdict": "supported_under_research_accounting_boundary",
"priority": "technical_backup",
"source_ids": [
"UP-S05"
],
"pairs": [
{
"experiment": "PRM_search",
"adaptive_generations": 16,
"baseline_generations": 64,
"comparison": "nearly matches or exceeds baseline in reported low-budget comparison"
},
{
"experiment": "revision",
"adaptive_generations": 64,
"baseline_generations": 256,
"comparison": "outperforms baseline in reported comparison"
}
],
"caveats": [
"Difficulty-estimation overhead excluded; generating 2048 samples to estimate difficulty can dominate savings",
"Specific fine-tuned models and 500 MATH test questions",
"Research generation budgets, not production monetary savings",
"No exact curve accuracy values digitised"
],
"chart_data": "2026-10-22-adaptive-budget-comparisons.csv"
},
{
"id": "UP05",
"claim": "The checked official 2026 AI Index does not provide a refreshed fixed-capability token-price series; its inference section concerns energy/carbon per query.",
"verdict": "supported_as_scoped_report_inspection",
"priority": "provenance_note",
"source_ids": [
"UP-S03"
],
"caveats": [
"Absence statement applies to this report check only",
"No denial that other newer price research exists"
]
},
{
"id": "UP06",
"claim": "A falling token price does not by itself imply a falling cost per accepted business task.",
"verdict": "accounting_inference_not_a_measured_spend_trend",
"priority": "main_message",
"source_ids": [
"UP-S01",
"UP-S02",
"UP-S04",
"UP-S05"
],
"formula": "Workflow cost per accepted result = total workflow cost / accepted results; token component = sum(input_tokens*input_rate + output_and_reasoning_tokens*output_rate) over calls; tool/infrastructure/review costs remain separate.",
"caveats": [
"No claim that every bill must rise",
"No assertion that Knauf currently samples64answers",
"Fixed workload can directly benefit from falling unit prices",
"Model-specific billable reasoning and cache treatment must be measured rather than assumed"
]
}
],
"rejected_wordings": [
{
"wording": "280x cheaper in 18 months",
"reason": "Endpoint dates imply 673 days, around 22.1 average months; Stanford's duration prose conflicts."
},
{
"wording": "AI bills must keep rising despite cheaper tokens",
"reason": "No universal enterprise spend claim established by these sources."
},
{
"wording": "64 samples costs 64 times more",
"reason": "No per-configuration token/latency/caching/aggregation ledger."
},
{
"wording": "Adaptive reasoning cuts production AI cost 75%",
"reason": "Research compute proxy excludes difficulty estimation and is workload-specific."
}
]
}