2026-10-22-agent-economics-claims
{
"schemaVersion": 1,
"retrievedOn": "2026-10-22",
"asOfDate": "2026-10-22",
"purpose": "Scoped public-primary-source claim audit for Knauf AI cost optimisation; no experimental reproduction",
"quotesStored": false,
"claims": [
{
"id": "AG01",
"placement": "main-candidate",
"slideClaim": "Agentic architectures can multiply token demand.",
"approvedCaption": "Anthropic observed ~4× tokens for agents and ~15× for multi-agent systems versus chats in its own data.",
"metric": "total tokens relative to chat interactions",
"chartData": [
{
"label": "Chat",
"value": 1
},
{
"label": "Agent",
"value": 4,
"approximate": true
},
{
"label": "Multi-agent",
"value": 15,
"approximate": true
}
],
"source": {
"title": "How we built our multi-agent research system",
"publisher": "Anthropic",
"publishedOn": "2025-06-13",
"url": "https://www.anthropic.com/research/multi-agent-research-system",
"locator": "Benefits of a multi-agent system, paragraphs 5–7",
"type": "first-party engineering observation"
},
"supports": "Architecture and task decomposition materially change resource consumption.",
"doesNotSupport": "15× against single agents; universal or future bill multipliers; matched-task causal comparison.",
"methodLimitations": "Population, sample size and precise token-accounting protocol are undisclosed.",
"counterevidence": "The same post reports 90.2% improvement on an internal research evaluation; extra effort can create valuable capability. Do not infer a joint cost/quality experiment.",
"verdict": "Supported as an attributed, approximate observation only"
},
{
"id": "AG02",
"placement": "main-candidate",
"slideClaim": "More reasoning does not reliably buy a better outcome.",
"approvedCaption": "Higher reasoning effort produced equal or lower accuracy in 21 of 36 matched HAL comparisons.",
"metric": "model–scaffold–benchmark comparisons with no accuracy improvement",
"chartData": {
"noImprovement": 21,
"improvement": 15,
"total": 36
},
"source": {
"title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation",
"publisher": "Princeton-led research team",
"publishedOn": "2025-10-13",
"version": "arXiv:2510.11977v1",
"url": "https://arxiv.org/html/2510.11977v1",
"locator": "Figure 3; §4.1 item 5; Appendix A4",
"type": "independent benchmark study"
},
"method": "Four model pairs across fixed scaffolds/benchmarks: Claude 3.7, Sonnet 4, Opus 4.1 no/high reasoning; o4-mini low/high. Broader study: 21,730 rollouts, 9 benchmarks, 9 models.",
"doesNotSupport": "58% of tasks or spending wasted; causal effect of reasoning on arbitrary enterprise workloads.",
"counterevidence": "Accuracy and token use correlate positively on 6/9 benchmarks. Reasoning can help.",
"methodLimitations": "Incomplete evaluation matrix; small SWE-bench subset; provider settings differ. SWE-Agent cost accounting omits cache discounts, limiting its dollar comparisons; no reliable latency claim.",
"verdict": "Supported under the tested model–scaffold–benchmark settings"
},
{
"id": "AG03",
"placement": "backup-candidate",
"approvedCaption": "Context editing reduced token consumption by 84% in Anthropic's 100-turn web-search evaluation.",
"metric": "token consumption relative to no context editing",
"chartData": [
{
"label": "Without editing",
"value": 100
},
{
"label": "With editing",
"value": 16
}
],
"source": {
"title": "Managing context on the Claude Developer Platform",
"publisher": "Anthropic",
"publishedOn": "2025-09-29",
"url": "https://claude.com/resources/articles/context-management",
"locator": "Performance improvements with context management",
"type": "first-party internal evaluation"
},
"supports": "Workflow/context engineering can reduce consumption while enabling completion of long workflows.",
"doesNotSupport": "84% savings on arbitrary workloads or an 84% reduction in total bills.",
"methodLimitations": "Vendor internal web-search evaluation; sample count, detailed distribution and full protocol undisclosed. Completion compared with workflows exhausting context, not a quality-equivalent financial comparison.",
"verdict": "Supported as a scoped vendor example"
},
{
"id": "AG04",
"placement": "backup-candidate",
"approvedCaption": "Repeated coding-agent runs on the same issue showed substantial cost variability.",
"metric": "max/min token-cost ratio across four independent runs, averaged across issues",
"chartData": {
"approximateMeanMaxMinRatio": 2,
"exactPerModelNumbersAvailable": false
},
"source": {
"title": "How Do AI Agents Spend Your Money? Analyzing and Predicting Token Consumption in Agentic Coding Tasks",
"publisher": "Bai et al.; Stanford/Michigan and collaborators",
"publishedOn": "2026-04-24",
"revisedOn": "2026-10-02",
"version": "arXiv:2604.22750v3",
"url": "https://arxiv.org/html/2604.22750v3",
"locator": "§2; §3 Figure2; §7 limitations",
"type": "working paper"
},
"method": "OpenHands, 500 SWE-bench Verified issues, eight models, four runs/issue/model; full history carried forward unchanged.",
"supports": "Budget distributions and runtime limits matter even with fixed tasks/models.",
"doesNotSupport": "30× is typical; causal harm from extra tokens; equivalent-workload 1000× demand comparison.",
"methodLimitations": "One scaffold/domain; association rather than randomized effort intervention; recent preprint. The 30× headline is an extreme; approximately 2× is the repeated-run summary.",
"verdict": "Supported within this coding-agent setup; use approximate variability or verbal explanation"
},
{
"id": "AG05",
"placement": "backup-candidate",
"approvedCaption": "Cost-aware prompt optimisation lowered variable inference cost while preserving benchmark accuracy.",
"metric": "variable inference cost versus default DSPy implementations",
"chartData": [
{
"model": "GPT-3.5",
"reductionPercent": 53
},
{
"model": "Llama-3-70B",
"reductionPercent": 41
}
],
"source": {
"title": "AI Agents That Matter",
"publisher": "Kapoor et al., Princeton University",
"publishedOn": "2024-07-01",
"version": "arXiv:2407.01502v1",
"url": "https://arxiv.org/html/2407.01502v1",
"locator": "§3.1–3.2; Figure2; Appendix B",
"type": "independent benchmark study; journal publication reported 2025"
},
"method": "HotPotQA multi-hop retrieval, 100 optimisation examples, 200 evaluation examples; five repetitions. Optimise temperature and few-shot selection/count/format.",
"doesNotSupport": "Same savings on current models or Knauf; end-to-end business cost reduction.",
"counterevidence": "One-time optimisation adds fixed cost. Study reports total-cost break-even around 1,350 tasks, specific to its setup.",
"verdict": "Supported as a historical proof of mechanism, not a current savings forecast"
},
{
"id": "AG06",
"placement": "backup-alternative",
"approvedCaption": "Anthropic's 2026 guidance reports multi-agent implementations typically using 3–10× tokens versus single-agent approaches on equivalent tasks in its testing.",
"metric": "tokens versus single-agent approach",
"source": {
"title": "Building multi-agent systems: When and how to use them",
"publisher": "Anthropic",
"publishedOn": "2026-01-23",
"url": "https://claude.com/resources/articles/building-multi-agent-systems-when-and-how-to-use-them",
"locator": "The case for starting with a single agent",
"type": "first-party engineering guidance"
},
"supports": "Duplicated context, coordination and handoffs can create material overhead.",
"doesNotSupport": "Same baseline as AG01 or proof of its multipliers; universal model bills.",
"methodLimitations": "No disclosed sample size, workload mix, token-accounting protocol or task-level table.",
"counterevidence": "The article identifies context isolation, parallelism and specialisation as cases where extra agents can justify their overhead.",
"verdict": "Supported as attributed vendor guidance; avoid displaying simultaneously with AG01"
}
],
"heldBackSources": [
{
"title": "The Price Reversal Phenomenon: When Cheaper Reasoning Models Cost More",
"url": "https://arxiv.org/html/2603.23971v2",
"version": "2026-05-28 v2",
"finding": "106/336 model-pair/dataset comparisons reverse input-plus-output list-price ranking and observed token-charge ranking.",
"reasonHeldBack": "Explicitly excludes quality; composite list-price proxy is workload-neutral; scope is 12 datasets, not enterprise procurement. Source prose says one in five despite 32%. Abstract 28× maximum should not be promoted without reconciling body examples. v1 has different models/tasks and 21.8%; never mix versions.",
"use": "Mechanism corroboration in research notes only"
},
{
"title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
"url": "https://arxiv.org/abs/2406.12045",
"reasonHeldBack": "Original 2024 model results are unsuitable as current 2026 capability estimates. pass^k measures all-k reliability, not at-least-one-success retry pass@k."
}
],
"accountingRule": "Token counts are not bills. Reprice uncached input, cache writes, cache reads, output/reasoning separately; add paid tools, hosting and review as appropriate. Compare cost per accepted outcome at fixed quality/latency constraints.",
"generalisationRule": "These sources establish mechanisms and workload-specific observations, not the future level of Knauf spend or a guaranteed savings percentage.",
"humanReviewFinding": "No transferable numerical review-cost estimate established in this audit; measure actual reviewer minutes and acceptance/rework locally."
}