2026-10-26-ai-benefit-source-register
{
"report_title": "Adversarial evidence on absent, negative, and overstated AI benefits",
"register_date": "2026-10-26",
"search_scope": "External-web research beginning with the assigned TechCrunch article; emphasis on business returns, productivity, adoption, rework, reliability, skill/workload effects, macro evidence, classical AI distinctions and promotional measurement.",
"source_record_count": 41,
"distinct_empirical_evidence_families": 34,
"deduplication": "S02 and S03 share Apollo aggregate evidence; S11 and S12 share the METR coding study family; S21 and S22 are Stack Overflow survey waves; S32 and S33 concern the same UC Berkeley ethnography. S01, S40 and S41 are contextual reporting/enforcement/commentary records, not separate productivity studies.",
"access_labels": "Full text read means the linked article/paper/report was reviewed. Abstract/summary only and access limited are stated per record; no paywall bypass was attempted. Figures and claims are attributed to the source and remain subject to each record's limits.",
"sources": [
{
"id": "S01",
"title": "TechCrunch report on Apollo's AI-return claim",
"date": "2026-10-25",
"source_urls": [
"https://techcrunch.com/2026/10/25/ai-delivers-zero-financial-payoff-to-most-companies-according-to-apollo-economist/"
],
"source_type": "secondary journalism",
"evidence_family_id": null,
"evidence_role": "Seed reporting; does not add a separately measured sample.",
"population_context": "U.S. public-company aggregate comparison discussed by Apollo economist Torsten Slok; excludes most nonpublic businesses.",
"method_sample": "Article relays Apollo charts/claims and an economist interview; no independent study sample.",
"verified_finding": "Reports flat revenue per worker and profit margins for S&P 493 versus gains for the Magnificent Seven, and no clear economy-wide productivity signal.",
"measured_outcome": "Aggregate accounting/productivity indicators as reported by Apollo; article itself is not causal evidence.",
"evidence_status": "contextual reporting; no causal estimate",
"limits": "Aggregate descriptive indicators cannot attribute changes to AI; article's broad phrasing is stronger than the underlying charts can establish.",
"access_status": "Full article text read.",
"commercial_or_promotion": "Journalism; underlying Apollo note is separately registered."
},
{
"id": "S02",
"title": "Apollo Daily Spark note on AI payoff outside technology",
"date": "2026-10-24",
"source_urls": [
"https://www.apollo.com/wealth/insights-news/insights/daily-spark/no-signs-of-ai-payoff-outside-the-tech-sector"
],
"source_type": "investment research commentary with descriptive charts",
"evidence_family_id": "F01_APOLLO_AGGREGATES",
"evidence_role": "Primary chart commentary; same evidence family as chart image record S03.",
"population_context": "Mag 7 versus S&P 493 public-company aggregates; labor productivity indicators; Ramp AI-usage data.",
"method_sample": "Descriptive aggregation using Bloomberg/Macrobond/Apollo, FRBSF utilization-adjusted TFP and BLS, plus Ramp AI Index; no treatment/control design.",
"verified_finding": "S&P 493 revenue per employee and profit margins were broadly flat in the displayed period while Mag 7 measures rose; charts show no clear AI-era break in aggregate productivity. The note describes AI usage adoption outside tech as limited.",
"measured_outcome": "Revenue per employee, margins, TFP/productivity proxies, reported firm AI usage.",
"evidence_status": "no clear aggregate signal; causal effect not measured",
"limits": "No downloadable chart series/counterfactual or AI attribution; sector composition, prices, capital and other trends may drive indicators.",
"access_status": "Full note and chart credits read.",
"commercial_or_promotion": "Apollo asset-management research; investment commentary, not an independent causal study."
},
{
"id": "S03",
"title": "Apollo chart-image series (five charts)",
"date": "2026-10-24",
"source_urls": [
"https://www.apollo.com/content/dam/apolloaem/images/daily-spark/2026/oct/24/Oct24_Chart1.png",
"https://www.apollo.com/content/dam/apolloaem/images/daily-spark/2026/oct/24/Oct24_Chart2.png",
"https://www.apollo.com/content/dam/apolloaem/images/daily-spark/2026/oct/24/Oct24_Chart3.png",
"https://www.apollo.com/content/dam/apolloaem/images/daily-spark/2026/oct/24/Oct24_Chart4.png",
"https://www.apollo.com/content/dam/apolloaem/images/daily-spark/2026/oct/24/Oct24_Chart5.png"
],
"source_type": "chart images",
"evidence_family_id": "F01_APOLLO_AGGREGATES",
"evidence_role": "Charts supporting S02; not an independent dataset or study.",
"population_context": "Public-company aggregate, productivity series and Ramp AI usage series as described in S02.",
"method_sample": "Five graphic series with credits and axes; no downloadable underlying time series.",
"verified_finding": "Charts visualize the comparisons summarized in S02; their displayed indicators do not identify AI as the cause of any change.",
"measured_outcome": "Descriptive financial and macroeconomic indicators.",
"evidence_status": "no causal estimate",
"limits": "Image-only figures, limited chart-level resolution, no firm-level counterfactual.",
"access_status": "All five images opened/read.",
"commercial_or_promotion": "Apollo publication assets."
},
{
"id": "S04",
"title": "NBER Working Paper 34836, Firm Data on AI",
"date": "2026",
"source_urls": [
"https://www.nber.org/papers/w34836",
"https://www.atlantafed.org/-/media/Project/Atlanta/FRBA/Documents/research/publication/working-paper/2026/03/24/03-firm-data-on-ai.pdf"
],
"source_type": "working paper and executive survey",
"evidence_family_id": "F02_FIRM_DATA_AI",
"evidence_role": "Original multi-country executive survey.",
"population_context": "Executives and employees in the United States, United Kingdom, Germany and Australia.",
"method_sample": "Nearly 6,000 executives, stratified surveys fielded Nov 2025–Jan 2026; related employee sample. Executive self-reports and forecasts.",
"verified_finding": "About 69% of firms reported active AI use; executive respondents averaged about 1.5 hours of own AI use per week and roughly one quarter reported none. Around nine in ten firms reported no productivity or employment effect over the previous three years. Executives forecast +1.4% productivity, +0.8% output and -0.7% employment over three years.",
"measured_outcome": "Reported firm impacts and expectations, not audited realized output or causal productivity.",
"evidence_status": "no reported past effect; prospective gains not yet measured",
"limits": "Self-reported adoption/impacts and forecasts; short exposure window; executive and employee responses differ.",
"access_status": "NBER abstract and full working-paper PDF via Federal Reserve Bank of Atlanta read.",
"commercial_or_promotion": "Academic/Federal Reserve research."
},
{
"id": "S05",
"title": "U.S. Census CES Working Paper 26-25, AI Use in Businesses",
"date": "2026",
"source_urls": [
"https://www.census.gov/library/working-papers/2026/adrm/CES-WP-26-25.html"
],
"source_type": "official business survey working paper",
"evidence_family_id": "F03_CENSUS_AI_MICROSTRUCTURE",
"evidence_role": "Original analysis of Census survey data.",
"population_context": "U.S. businesses in the Business Trends and Outlook Survey.",
"method_sample": "AI supplement fielded Nov 2025–Jan 2026; firm and employment-weighted adoption and usage breadth.",
"verified_finding": "About 18% of firms reported AI use in a business function (32% employment-weighted); 23% reported task use (41% employment-weighted). 57% of adopter firms used AI in three or fewer functions and 65% in three or fewer tasks. Only about 2% reported AI-related employment decline. Breadth correlated positively with commercial performance.",
"measured_outcome": "Self-reported adoption breadth, commercial performance and employment changes.",
"evidence_status": "limited-depth use; positive association, not causal",
"limits": "Cross-sectional self-report; correlation cannot show AI caused performance; definition differs across function/task measures.",
"access_status": "Full working-paper page read.",
"commercial_or_promotion": "U.S. Census Bureau working paper; not an endorsement of causal interpretation."
},
{
"id": "S06",
"title": "McKinsey 2025 State of AI survey",
"date": "2025",
"source_urls": [
"https://www.mckinsey.com/capabilities/quantumblack/our-insights/the-state-of-ai-2025"
],
"source_type": "consulting-firm survey",
"evidence_family_id": "F04_MCKINSEY_2025",
"evidence_role": "Commercial survey; not independent administrative measurement.",
"population_context": "Global organizations and executives.",
"method_sample": "Online survey n=1,993 across 105 countries; fielded 25 June–29 July 2025.",
"verified_finding": "88% reported regular AI use in at least one function. Only 39% reported any enterprise-level EBIT impact; most of those reported less than 5% EBIT contribution. About 6% met the report's high-performer definition of at least 5% EBIT contribution and significant value.",
"measured_outcome": "Respondent-reported use and AI-attributed EBIT.",
"evidence_status": "most respondents reported no enterprise EBIT impact; small high-performing subgroup",
"limits": "Self-reported estimates and definitions; nonprobability survey; attribution to AI not audited.",
"access_status": "Full survey report page read.",
"commercial_or_promotion": "McKinsey & Company proprietary survey; commissioning and consulting interest disclosed."
},
{
"id": "S07",
"title": "McKinsey 2026 State of AI survey",
"date": "2026",
"source_urls": [
"https://www.mckinsey.com/capabilities/quantumblack/our-insights/the-state-of-ai"
],
"source_type": "consulting-firm survey",
"evidence_family_id": "F05_MCKINSEY_2026",
"evidence_role": "New survey wave, distinct from S06.",
"population_context": "Global business respondents.",
"method_sample": "Online survey n=1,719 in 97 countries; fielded 4 May–8 June 2026.",
"verified_finding": "80% said AI improved their own productivity; 37% reported any positive EBIT impact, described as flat versus the previous survey. Negative effects or strain were reported by 47% of middle managers/individual contributors versus 31% of executives.",
"measured_outcome": "Self-reported personal productivity, EBIT attribution and strain.",
"evidence_status": "mixed self-report; enterprise financial effect reported by minority",
"limits": "Not audited financial data or a causal design; respondent perceptions and subgroup differences.",
"access_status": "Full survey report page read.",
"commercial_or_promotion": "McKinsey & Company proprietary survey."
},
{
"id": "S08",
"title": "S&P Global Market Intelligence/451 Research 2025 GenAI survey",
"date": "2025-10",
"source_urls": [
"https://www.spglobal.com/market-intelligence/en/news-insights/research/2025/10/generative-ai-shows-rapid-growth-but-yields-mixed-results"
],
"source_type": "commercial technology-market survey",
"evidence_family_id": "F06_SP451_ENTERPRISE",
"evidence_role": "Survey of active enterprise technology decision makers.",
"population_context": "North American and European mid/senior IT and line-of-business professionals.",
"method_sample": "Survey of 1,006 professionals; report compares GenAI deployment and abandoned proof-of-concept rates.",
"verified_finding": "Respondents reporting most initiatives abandoned before production rose from 17% to 42%; organizations reported scrapping an average 46% of proof-of-concepts pre-production. 46% reported no strong positive enterprise impact and 19% strong impact across most areas.",
"measured_outcome": "Respondent-reported pilot disposition and perceived impact.",
"evidence_status": "mixed; substantial reported pilot abandonment",
"limits": "Survey wording and respondent pool; abandoning a pilot is not necessarily failure, and impact is self-reported.",
"access_status": "Full report summary/page read.",
"commercial_or_promotion": "S&P Global Market Intelligence/451 Research survey product."
},
{
"id": "S09",
"title": "MIT NANDA, The GenAI Divide: State of AI in Business 2025",
"date": "2025-07",
"source_urls": [
"https://pureinsights.com/wp-content/uploads/2025/09/MIT-Report-v0.1_State_of_AI_in_Business_2025_Report.pdf"
],
"source_type": "research-group report using interviews and case review",
"evidence_family_id": "F07_MIT_NANDA",
"evidence_role": "Large headline estimate with explicit methodological caveats.",
"population_context": "Enterprise AI initiatives across sectors; interviews and conference-attending leaders.",
"method_sample": "Review of 300+ public initiatives, 52 organizational interviews and 153 senior leaders at four conferences, Jan–Jun 2025.",
"verified_finding": "Report says roughly 60% of enterprise tools surveyed were explored, 20% piloted and 5% reached production; headline says 95% of organizations see zero return while 5% pilots generate millions.",
"measured_outcome": "Case-review and interview classifications of production and perceived sustained productivity/P&L.",
"evidence_status": "reported zero return for most cases; not a representative population estimate",
"limits": "Authors state directionally derived interview figures, varying sample sizes and success definitions, anonymized underlying data; not official audited reporting.",
"access_status": "Full PDF read, including limitations.",
"commercial_or_promotion": "MIT NANDA research report; headline is more categorical than its stated sampling supports."
},
{
"id": "S10",
"title": "Atlassian 2025 AI Collaboration Report",
"date": "2025",
"source_urls": [
"https://www.atlassian.com/blog/cxo-ai-collaboration-report-2025.pdf"
],
"source_type": "vendor-commissioned survey",
"evidence_family_id": "F08_ATLASSIAN_COLLABORATION",
"evidence_role": "Executive and worker self-report.",
"population_context": "Fortune 1000 executives and knowledge workers.",
"method_sample": "180 executives and approximately 12,000 knowledge workers.",
"verified_finding": "96% of surveyed executives said their organizations had not achieved meaningful ROI; 4% reported dramatic gains in efficiency, innovation or quality. Workers reported 33% personal productivity improvement and 1.3 hours/day saved.",
"measured_outcome": "Perceived ROI and self-reported individual productivity/time.",
"evidence_status": "mixed; high reported personal gains alongside low perceived enterprise ROI",
"limits": "Self-report and vendor survey design; not audited ROI or time-use tracking.",
"access_status": "Full PDF read.",
"commercial_or_promotion": "Atlassian-sponsored report."
},
{
"id": "S11",
"title": "METR randomized study of experienced open-source developers",
"date": "2025-07",
"source_urls": [
"https://metr.org/Early_2025_AI_Experienced_OS_Devs_Study-paper.pdf"
],
"source_type": "randomized controlled field study",
"evidence_family_id": "F09_METR_CODING",
"evidence_role": "Primary coding productivity experiment.",
"population_context": "Experienced developers working in mature open-source repositories.",
"method_sample": "16 developers completed 246 real issues; tasks randomized to AI allowed or disallowed, mainly Cursor Pro with Claude 3.5/3.7 Sonnet.",
"verified_finding": "AI-allowed tasks took about 19% longer on average (95% CI approximately +2% to +39%); developers had expected a 24% speedup before and estimated a 20% speedup after.",
"measured_outcome": "Task completion time in real repositories.",
"evidence_status": "negative in this bounded study",
"limits": "Small sample, experienced OSS developers, narrow tools/tasks and early-2025 models; not representative of all coding work.",
"access_status": "Full paper PDF read.",
"commercial_or_promotion": "METR nonprofit research."
},
{
"id": "S12",
"title": "METR February 2026 uplift follow-up",
"date": "2026-02-24",
"source_urls": [
"https://metr.org/blog/2026-02-24-uplift-update/"
],
"source_type": "follow-up analysis of the METR coding experiment program",
"evidence_family_id": "F09_METR_CODING",
"evidence_role": "Same research family as S11; follow-up cautions that later evidence is unreliable.",
"population_context": "Experienced developers in mature OSS repositories.",
"method_sample": "Reassesses original participant estimates and newly recruited sample amid opt-out/selection and timing issues.",
"verified_finding": "Original participants later estimated AI made tasks 18% faster (CI −38% to +9%); newly recruited developers estimated 4% faster (CI −15% to +9%). Authors said selection and timing issues made the follow-up data unreliable.",
"measured_outcome": "Developer-estimated time uplift; not a clean randomized replication.",
"evidence_status": "uncertain/no detectable effect in the follow-up",
"limits": "Selection into AI/no-AI conditions, lower compensation for no-AI tasks and multi-agent timing undermine causal inference.",
"access_status": "Full update read.",
"commercial_or_promotion": "METR nonprofit research."
},
{
"id": "S13",
"title": "Cui et al. three-company coding-assistant field experiments",
"date": "2026-02-27",
"source_urls": [
"https://pubsonline.informs.org/doi/10.1287/mnsc.2025.00535"
],
"source_type": "multi-company randomized field experiments",
"evidence_family_id": "F10_CUI_CODING_RCTS",
"evidence_role": "Distinct positive coding experiment family.",
"population_context": "Developers at Microsoft, Accenture and an anonymous Fortune 100 company.",
"method_sample": "Three randomized experiments combined across 4,867 developers; published in Management Science.",
"verified_finding": "Combined estimate: 26.08% increase in tasks completed, standard error 10.3%; adoption and gains were larger among less experienced developers; individual experiments varied and were noisy.",
"measured_outcome": "Task completion counts, not profit or long-term code quality.",
"evidence_status": "positive but bounded",
"limits": "Employer/tool/task-specific; high uncertainty and heterogeneous trials; quality, downstream rework and enterprise financial return not established.",
"access_status": "Publisher page and study details read; access to full article limited.",
"commercial_or_promotion": "Academic paper; Microsoft/Accenture field settings."
},
{
"id": "S14",
"title": "DORA 2024 Accelerate State of DevOps report",
"date": "2024",
"source_urls": [
"https://dora.dev/research/2024/dora-report/",
"https://dora.dev/research/2024/dora-report/2024-dora-accelerate-state-of-devops-report.pdf"
],
"source_type": "observational developer survey and statistical model",
"evidence_family_id": "F11_DORA_2024",
"evidence_role": "Large observational evidence on AI adoption and software delivery.",
"population_context": "Technology professionals and organizations globally.",
"method_sample": "More than 39,000 survey respondents; modeled relationships between AI adoption and software-delivery outcomes.",
"verified_finding": "Report associates AI adoption with individual productivity, flow and satisfaction, but estimates a 25% increase in AI adoption predicts 1.5% lower delivery throughput and 7.2% lower stability.",
"measured_outcome": "Self-reported developer experience plus modeled software-delivery metrics.",
"evidence_status": "mixed associations",
"limits": "Observational modeling; report's interval has 89% uncertainty; association is not proof of AI causation.",
"access_status": "Full report and PDF read.",
"commercial_or_promotion": "DORA/Google Cloud research program."
},
{
"id": "S15",
"title": "Brynjolfsson, Li and Raymond, Generative AI at Work",
"date": "2023",
"source_urls": [
"https://www.nber.org/papers/w31161"
],
"source_type": "staggered rollout field study",
"evidence_family_id": "F12_SUPPORT_AGENTS",
"evidence_role": "Customer-support productivity study with heterogeneous effects.",
"population_context": "Customer-support agents at a large software firm.",
"method_sample": "Approximately 5,179 agents; phased rollout provided AI access to about 1,300.",
"verified_finding": "AI access raised issues resolved per hour by 14% on average, up to 35% for least-skilled/least-experienced workers; top-skilled gains were negligible and some customer-satisfaction measures fell slightly.",
"measured_outcome": "Issues resolved/hour, with customer satisfaction measures.",
"evidence_status": "positive average with heterogeneous and adverse subgroup outcomes",
"limits": "One firm and workflow; rollout design; no direct profit estimate.",
"access_status": "NBER paper page and abstract read; full paper access limited.",
"commercial_or_promotion": "Academic research; original company setting anonymized."
},
{
"id": "S16",
"title": "Noy and Zhang, Experimental Evidence on the Productivity Effects of Generative AI",
"date": "2023-07",
"source_urls": [
"https://pubmed.ncbi.nlm.nih.gov/37440646/"
],
"source_type": "randomized task experiment",
"evidence_family_id": "F13_WRITING_TASK_RCT",
"evidence_role": "Controlled writing-task productivity evidence.",
"population_context": "College-educated professionals completing occupation-specific writing assignments.",
"method_sample": "453 participants randomized to ChatGPT access or control.",
"verified_finding": "ChatGPT reduced task time by 40% and increased evaluated output quality by 18% in the experimental tasks.",
"measured_outcome": "Time and blinded quality ratings for short writing tasks.",
"evidence_status": "positive in bounded task setting",
"limits": "Online experiment, short-form work and no organizational integration, rework or financial outcomes.",
"access_status": "PubMed abstract read; journal full text not read.",
"commercial_or_promotion": "Peer-reviewed research."
},
{
"id": "S17",
"title": "Dell'Acqua et al., Navigating the Jagged Technological Frontier",
"date": "2026",
"source_urls": [
"https://pubsonline.informs.org/doi/10.1287/orsc.2025.21838",
"https://www.hbs.edu/ris/Publication%20Files/dell-acqua-et-al-2026-navigating-the-jagged-technological-frontier_5c589c8c-fbb5-458f-b285-c944746cd717.pdf"
],
"source_type": "randomized field experiment",
"evidence_family_id": "F14_CONSULTANT_FRONTIER",
"evidence_role": "Task-dependent gains and errors in professional work.",
"population_context": "Management consultants completing consulting tasks.",
"method_sample": "758 consultants; GPT-4 assistance tested on tasks inside and outside model capability frontier.",
"verified_finding": "Within-frontier tasks improved speed/quality; on an outside-frontier task, unaided control answers were correct 84.5% versus 60% and 70.6% in AI conditions, roughly a 19–25 percentage-point decline depending on condition.",
"measured_outcome": "Task time, quality and correctness.",
"evidence_status": "mixed; performance depends on task/model capability frontier",
"limits": "One profession, model and task suite; does not establish long-run adoption or firm financial returns.",
"access_status": "Full HBS PDF read; publisher page read.",
"commercial_or_promotion": "Academic/HBS field experiment."
},
{
"id": "S18",
"title": "Fitzpatrick et al. public-sector GenAI field experiment",
"date": "2025-02",
"source_urls": [
"https://arxiv.org/abs/2502.09479"
],
"source_type": "preregistered field experiment preprint",
"evidence_family_id": "F15_PUBLIC_SECTOR_TASKS",
"evidence_role": "Mixed task-level public-sector evidence.",
"population_context": "Public-sector workers completing document and data tasks.",
"method_sample": "Preregistered experiment; abstract reports task categories but does not provide sample size in the accessible material read.",
"verified_finding": "Document task quality rose 17% and time fell 34%; data task quality fell 12%, with no significant time change.",
"measured_outcome": "Task-level quality and completion time.",
"evidence_status": "mixed across tasks",
"limits": "Preprint; abstract/summary only; sample details and generalizability not verified from full text.",
"access_status": "Abstract/summary read only.",
"commercial_or_promotion": "Academic preprint."
},
{
"id": "S19",
"title": "NBER Working Paper 33795, Shifting Work Patterns with Generative AI",
"date": "2025",
"source_urls": [
"https://www.nber.org/papers/w33795",
"https://www.microsoft.com/en-us/research/publication/shifting-work-patterns-with-generative-ai/"
],
"source_type": "randomized workplace access study",
"evidence_family_id": "F16_WORK_PATTERNS_RCT",
"evidence_role": "Work-time and task-mix study.",
"population_context": "Knowledge workers across 66 firms.",
"method_sample": "7,137 workers followed for six months; randomized access to a GenAI tool.",
"verified_finding": "Frequent users reported 3.6 fewer hours of email per week; intention-to-treat effect was 1.3 hours. Documents were completed moderately faster; meeting time did not significantly change; no detectable shift in task quantity or composition.",
"measured_outcome": "Self-reported time allocation and work patterns, not output or financial returns.",
"evidence_status": "time savings in some activities; no detected broader task shift",
"limits": "Tool access and self-reported time; no direct quality, output or profit measurement; single employer ecosystem.",
"access_status": "NBER and Microsoft Research pages read; study summary available.",
"commercial_or_promotion": "Academic/Microsoft Research collaboration."
},
{
"id": "S20",
"title": "Humlum and Vestergaard, Still Waters, Rapid Currents",
"date": "2025",
"source_urls": [
"https://www.nber.org/papers/w33777"
],
"source_type": "worker survey linked to administrative records, difference-in-differences",
"evidence_family_id": "F17_DENMARK_LABOR",
"evidence_role": "Labor-market effects of occupational GenAI exposure.",
"population_context": "Danish workers in 11 GenAI-exposed occupations.",
"method_sample": "About 25,000 workers; worker surveys linked to administrative earnings and hours records.",
"verified_finding": "No detectable average effect on earnings or recorded hours roughly two years after ChatGPT; authors report ruling out average effects larger than about 2%. Workers reported reorganized tasks and new AI-related tasks.",
"measured_outcome": "Earnings, recorded hours and survey-reported tasks.",
"evidence_status": "no detectable average earnings/hours effect; task change reported",
"limits": "Country/occupation-specific and early adoption period; null average effect does not imply no individual or future effect.",
"access_status": "NBER landing page/abstract read; working paper full text inaccessible.",
"commercial_or_promotion": "Academic research."
},
{
"id": "S21",
"title": "Stack Overflow 2024 Developer Survey: AI",
"date": "2024",
"source_urls": [
"https://survey.stackoverflow.co/2024/ai"
],
"source_type": "self-selected developer survey",
"evidence_family_id": "F18_STACKOVERFLOW_AI_SURVEYS",
"evidence_role": "First wave in recurring Stack Overflow survey series.",
"population_context": "Software developers responding to Stack Overflow annual survey.",
"method_sample": "AI-use question base about 60,907 respondents; online voluntary survey.",
"verified_finding": "Professional developers reported about 41% trust in AI accuracy; 66% said distrust or near-correct output was a challenge, and 12% said AI created more review work.",
"measured_outcome": "Self-reported trust and review burden.",
"evidence_status": "reported verification/review concern",
"limits": "Self-selected respondents, perceptions rather than tracked defects or time; not a productivity estimate.",
"access_status": "Full survey page read.",
"commercial_or_promotion": "Stack Overflow platform survey; self-selected sample."
},
{
"id": "S22",
"title": "Stack Overflow 2025 Developer Survey: AI",
"date": "2025",
"source_urls": [
"https://survey.stackoverflow.co/2025/ai"
],
"source_type": "self-selected developer survey",
"evidence_family_id": "F18_STACKOVERFLOW_AI_SURVEYS",
"evidence_role": "Second wave of same recurring survey family as S21.",
"population_context": "Global software developers responding to Stack Overflow annual survey.",
"method_sample": "More than 49,000 respondents across 177 countries; online voluntary survey.",
"verified_finding": "84% reported using or planning to use AI tools; 46% distrusted AI accuracy versus 33% who trusted it.",
"measured_outcome": "Self-reported adoption intention/use and trust.",
"evidence_status": "reported distrust amid high use/planned use",
"limits": "Self-selection and self-report; use and trust are not measured productivity or defect rates.",
"access_status": "Full survey page read.",
"commercial_or_promotion": "Stack Overflow platform survey; not an independent replication of S21."
},
{
"id": "S23",
"title": "GitClear 2025 AI Copilot Code Quality report",
"date": "2025",
"source_urls": [
"https://gitclear-public.s3.us-west-2.amazonaws.com/GitClear-AI-Copilot-Code-Quality-2025.pdf"
],
"source_type": "commercial code-analytics report",
"evidence_family_id": "F19_GITCLEAR_CODE_QUALITY",
"evidence_role": "Longitudinal code-change analysis.",
"population_context": "Software repository commits in GitClear's analyzed dataset.",
"method_sample": "Analysis of 211 million changed lines, Jan 2020–Dec 2024; code authorship by AI not reliably identified.",
"verified_finding": "Report found duplication/copy-paste and code-churn indicators rose relative to historical forecasts as coding assistants spread.",
"measured_outcome": "Repository change patterns and code churn proxies, not developer time or software defects directly.",
"evidence_status": "negative quality proxy trend; attribution uncertain",
"limits": "Observational; authorship and causal attribution uncertain; GitClear customer/data selection.",
"access_status": "Full PDF read.",
"commercial_or_promotion": "Vendor-produced analytics report."
},
{
"id": "S24",
"title": "Workday commissioned survey on AI rework",
"date": "2026-01-14",
"source_urls": [
"https://newsroom.workday.com/2026-01-14-New-Workday-Research-Companies-Are-Leaving-AI-Gains-on-the-Table"
],
"source_type": "vendor-commissioned user survey",
"evidence_family_id": "F20_WORKDAY_AI_REWORK",
"evidence_role": "Self-reported time savings and correction burden.",
"population_context": "Active AI users at larger organizations.",
"method_sample": "3,200 users at companies with at least $100 million annual revenue; Hanover Research fielded Nov 2025.",
"verified_finding": "85% reported saving 1–7 hours per week; nearly 40% of time saved was spent reworking or verifying AI output; 14% reported consistent net-positive outcomes.",
"measured_outcome": "Perceived weekly time saved/rework and net outcome.",
"evidence_status": "mixed; respondents report time saved and substantial rework",
"limits": "User-reported and vendor-commissioned; no time-use or audited ROI measurement.",
"access_status": "Full newsroom release read.",
"commercial_or_promotion": "Workday-commissioned survey; commercial vendor."
},
{
"id": "S25",
"title": "Zapier survey on AI workslop and cleanup",
"date": "2026-01-14",
"source_urls": [
"https://www.globenewswire.com/de/news-release/2026/01/14/3218545/0/en/Zapier-Survey-Finds-Workers-Spend-4-5-Hours-Per-Week-Cleaning-Up-AI-Mistakes.html"
],
"source_type": "vendor-sponsored survey",
"evidence_family_id": "F21_ZAPIER_AI_CLEANUP",
"evidence_role": "AI user self-report on rework.",
"population_context": "U.S. employees at companies with 250+ staff who use AI at work.",
"method_sample": "1,100 AI users; Centiment survey fielded 13–14 Nov 2025; unweighted online sample.",
"verified_finding": "92% said AI boosts productivity; reported average cleanup of AI mistakes was 4.5 hours/week; 74% reported at least one negative consequence.",
"measured_outcome": "Self-reported productivity perception, rework time and consequences.",
"evidence_status": "mixed self-report; reported cleanup burden",
"limits": "Very short field period, selected AI users, vendor-sponsored, self-reported time.",
"access_status": "Full release read.",
"commercial_or_promotion": "Zapier-sponsored survey; commercial automation vendor."
},
{
"id": "S26",
"title": "BetterUp and Stanford Social Media Lab workslop survey",
"date": "2025-09",
"source_urls": [
"https://www.betterup.com/workslop"
],
"source_type": "commercial survey and modeled cost estimate",
"evidence_family_id": "F22_BETTERUP_WORKSLOP",
"evidence_role": "Recipient-reported low-quality AI work and rework.",
"population_context": "U.S. full-time desk workers.",
"method_sample": "1,150 online respondents fielded Aug–Sep 2025; BetterUp with Stanford Social Media Lab.",
"verified_finding": "40% said they received workslop in the prior month; respondents estimated 15.4% of work received fit the definition; each incident took about 1 hour 56 minutes to address.",
"measured_outcome": "Self-reported incidence and time; extrapolated labor cost is modeled, not audited.",
"evidence_status": "reported negative rework burden",
"limits": "Survey definition and recall; self-report; no objective output quality or enterprise financial measure.",
"access_status": "Full survey page read.",
"commercial_or_promotion": "BetterUp sells coaching; survey and modeled cost should be treated as commissioned/commercial evidence."
},
{
"id": "S27",
"title": "Magesh et al. legal research-tool hallucination evaluation",
"date": "2024-05",
"source_urls": [
"https://arxiv.org/abs/2405.20362"
],
"source_type": "benchmark evaluation preprint",
"evidence_family_id": "F23_LEGAL_RESEARCH_RELIABILITY",
"evidence_role": "Accuracy and hallucination benchmark in legal research.",
"population_context": "Legal-research queries answered by GPT-4 and commercial legal research assistants.",
"method_sample": "More than 200 legal queries; answers reviewed for accuracy and hallucinated citations/content.",
"verified_finding": "Reported answer accuracy was about 65% for Lexis AI, 42% for Westlaw AI and 19% for Ask Practical Law; hallucination rates ranged about 17–33%.",
"measured_outcome": "Benchmark answer accuracy and hallucination, not attorney time or case outcomes.",
"evidence_status": "negative reliability limits on evaluated tasks",
"limits": "Task/query set and scoring choices; benchmark results do not directly estimate law-firm productivity; preprint.",
"access_status": "Full arXiv paper read; publisher version inaccessible.",
"commercial_or_promotion": "Academic preprint; tool vendors not study sponsors."
},
{
"id": "S28",
"title": "OSWorld 2.0 computer-use agent benchmark",
"date": "2026-06",
"source_urls": [
"https://arxiv.org/abs/2606.29537"
],
"source_type": "agent benchmark preprint",
"evidence_family_id": "F24_OSWORLD_AGENTS",
"evidence_role": "Reliability evidence for multi-step computer-use agents.",
"population_context": "Agents executing desktop workflows across computer applications.",
"method_sample": "108 workflows; human baseline median 1.6 hours; agents evaluated over hundreds of steps and up to 500 steps.",
"verified_finding": "Best system achieved 20.6% strict success at 500 steps and 54.8% partial success; GPT-5.5 reached 13% strict success.",
"measured_outcome": "Benchmark task completion and partial completion.",
"evidence_status": "low strict reliability on evaluated workflows",
"limits": "Benchmark is not enterprise field deployment; workflow mix and agent versions change rapidly; partial success differs from business usefulness.",
"access_status": "Full arXiv paper read.",
"commercial_or_promotion": "Academic benchmark preprint."
},
{
"id": "S29",
"title": "Anthropic randomized study of AI assistance and coding skill",
"date": "2026-01",
"source_urls": [
"https://www.anthropic.com/research/AI-assistance-coding-skills",
"https://arxiv.org/abs/2601.20245"
],
"source_type": "randomized learning experiment",
"evidence_family_id": "F25_ANTHROPIC_CODING_SKILL",
"evidence_role": "Immediate learning outcome when AI is used during coding.",
"population_context": "Software developers learning an unfamiliar Python library.",
"method_sample": "52 developers randomized to AI assistance or unaided learning.",
"verified_finding": "AI group scored 17 percentage points lower on an immediate comprehension quiz; task-time reduction was small and not statistically significant.",
"measured_outcome": "Immediate quiz score and task time.",
"evidence_status": "negative immediate learning result; long-term effect unmeasured",
"limits": "Small sample, one unfamiliar library and short study; does not establish durable skill decline.",
"access_status": "Full Anthropic article and preprint read.",
"commercial_or_promotion": "Anthropic-authored study; company developer of tested AI."
},
{
"id": "S30",
"title": "Nature Human Behaviour 2026 management-consultant AI RCT",
"date": "2026-10-26",
"source_urls": [
"https://www.nature.com/articles/s41562-026-02566-6"
],
"source_type": "randomized controlled experiment",
"evidence_family_id": "F26_CONSULTANT_TECHNICAL_SKILL",
"evidence_role": "AI use during a technical task and later unaided performance.",
"population_context": "Management consultants completing a data-science task outside their usual expertise.",
"method_sample": "487 consultants; abstract reports treatment and later unaided task performance.",
"verified_finding": "AI-treated participants scored higher while using AI but remained below expert data scientists; no evidence of improved technical answers without AI afterward.",
"measured_outcome": "Task answer scores with and without AI.",
"evidence_status": "mixed; assisted uplift without detected transfer",
"limits": "Abstract-level evidence read; one task and occupation; no long-term skill measurement.",
"access_status": "Abstract only; full text not read.",
"commercial_or_promotion": "Peer-reviewed journal study."
},
{
"id": "S31",
"title": "Microsoft Research/CMU survey on critical thinking with GenAI",
"date": "2025",
"source_urls": [
"https://www.microsoft.com/en-us/research/publication/the-impact-of-generative-ai-on-critical-thinking-self-reported-reductions-in-cognitive-effort-and-confidence-effects-from-a-survey-of-knowledge-workers/"
],
"source_type": "observational survey",
"evidence_family_id": "F27_CRITICAL_THINKING_SURVEY",
"evidence_role": "Self-reported cognitive effort and trust.",
"population_context": "Knowledge workers using generative AI at work.",
"method_sample": "319 workers and 936 self-reported examples; survey study.",
"verified_finding": "Higher confidence/trust in AI was associated with less reported critical-thinking effort; confidence in one's own task skills was associated with more effort.",
"measured_outcome": "Self-reported cognitive effort and confidence.",
"evidence_status": "association only; no causal skill decline measured",
"limits": "Self-report and observational associations; does not objectively measure cognition or lasting skill change.",
"access_status": "Full research page read.",
"commercial_or_promotion": "Microsoft Research/Carnegie Mellon academic collaboration."
},
{
"id": "S32",
"title": "UC Berkeley Haas ethnography on AI work intensification",
"date": "2026-02",
"source_urls": [
"https://www.universityofcalifornia.edu/news/ai-promised-free-workers-time-uc-berkeley-haas-researchers-found-opposite"
],
"source_type": "qualitative ongoing ethnography",
"evidence_family_id": "F28_WORK_INTENSIFICATION_ETHNOGRAPHY",
"evidence_role": "Primary report on one-company ethnographic study.",
"population_context": "A 200-person U.S. technology company.",
"method_sample": "Eight months of observation and more than 40 interviews; ongoing study.",
"verified_finding": "Researchers described expanded work, faster pace and longer hours after AI tools entered workflows.",
"measured_outcome": "Qualitative work practices and worker experience.",
"evidence_status": "reported work intensification in one organization",
"limits": "Single company, qualitative and in-progress; no causal average or quantitative productivity measure.",
"access_status": "Full university news article read; underlying study not published in a complete paper.",
"commercial_or_promotion": "University reporting on an academic ethnography."
},
{
"id": "S33",
"title": "Harvard Business Review article on AI and work intensification",
"date": "2026-02",
"source_urls": [
"https://hbr.org/2026/02/ai-doesnt-reduce-work-it-intensifies-it"
],
"source_type": "secondary commentary/reporting",
"evidence_family_id": "F28_WORK_INTENSIFICATION_ETHNOGRAPHY",
"evidence_role": "Relays the same UC Berkeley ethnography as S32; not an independent sample.",
"population_context": "The same 200-person technology company discussed in S32.",
"method_sample": "Article discussion based on the ongoing eight-month ethnography and interviews; no new sample.",
"verified_finding": "Describes workload expansion and intensification rather than reduced work hours.",
"measured_outcome": "Qualitative work practices; secondary interpretation.",
"evidence_status": "same underlying evidence family as S32",
"limits": "No independent replication or additional measured effect; article access limited.",
"access_status": "Page/summary read; full article access limited.",
"commercial_or_promotion": "HBR editorial publication; not a separate empirical study."
},
{
"id": "S34",
"title": "Federal Reserve note on AI buildout and publicly available productivity indicators",
"date": "2026-07-17",
"source_urls": [
"https://www.federalreserve.gov/econres/notes/feds-notes/the-ai-buildout-and-the-economy-publicly-available-data-to-assess-ais-impact-20260717.html"
],
"source_type": "official macroeconomic indicator review",
"evidence_family_id": "F29_FED_MACRO_INDICATORS",
"evidence_role": "Aggregate productivity assessment.",
"population_context": "U.S. sectors with differing AI exposure and macroeconomic indicators.",
"method_sample": "Federal Reserve staff synthesis of publicly available adoption, investment and productivity series.",
"verified_finding": "High-, medium- and low-exposure sectors' productivity trends were relatively consistent; no large aggregate AI productivity signal was visible in available indicators.",
"measured_outcome": "Sector and aggregate productivity time series, not firm-level causal effects.",
"evidence_status": "no clear aggregate signal detected yet",
"limits": "Adoption may be shallow; lags, intangible investment measurement, sector trends and pre-existing differences complicate detection.",
"access_status": "Full note read.",
"commercial_or_promotion": "Federal Reserve staff analysis."
},
{
"id": "S35",
"title": "Kansas City Fed, A New U.S. Productivity Chapter?",
"date": "2026-02-11",
"source_urls": [
"https://www.kansascityfed.org/research/economic-bulletin/a-new-us-productivity-chapter-what-industry-data-say-about-ai/"
],
"source_type": "regional Federal Reserve industry analysis",
"evidence_family_id": "F30_KC_FED_PRODUCTIVITY",
"evidence_role": "Industry-level productivity and AI adoption association.",
"population_context": "U.S. industries since 2022.",
"method_sample": "Industry productivity and AI adoption comparisons.",
"verified_finding": "U.S. labor productivity has exceeded its prepandemic trend since 2022, but a small set of industries explains much of the increase; AI adoption correlates with faster productivity in some industries but explains little of the aggregate shift.",
"measured_outcome": "Industry productivity and reported adoption association.",
"evidence_status": "mixed; aggregate attribution limited",
"limits": "Correlation rather than causal identification; industry-level measures and adoption proxies.",
"access_status": "Full analysis read.",
"commercial_or_promotion": "Federal Reserve Bank of Kansas City analysis."
},
{
"id": "S36",
"title": "Acemoglu, The Simple Macroeconomics of AI",
"date": "2024-05",
"source_urls": [
"https://www.nber.org/papers/w32487",
"https://economics.mit.edu/sites/default/files/2024-05/The%20Simple%20Macroeconomics%20of%20AI.pdf"
],
"source_type": "economic model",
"evidence_family_id": "F31_ACEMOGLU_MACRO_MODEL",
"evidence_role": "Model-based estimate of potential aggregate productivity effect.",
"population_context": "U.S. economy over a ten-year horizon.",
"method_sample": "Task-based macroeconomic model using exposure, automation scope and cost savings assumptions.",
"verified_finding": "Model estimates no more than about 0.66% cumulative U.S. TFP increase over 10 years under its assumptions; author argues even that may be an upper bound.",
"measured_outcome": "Modeled potential TFP, not observed outcome.",
"evidence_status": "model predicts modest effect; not yet measured",
"limits": "Highly assumption-dependent, excludes or simplifies many future uses and adjustment effects; not an empirical forecast with known precision.",
"access_status": "Full MIT-hosted paper read; NBER page read.",
"commercial_or_promotion": "Academic economic analysis."
},
{
"id": "S37",
"title": "U.S. Census CES Working Paper 25-27 on industrial AI",
"date": "2025",
"source_urls": [
"https://www2.census.gov/library/working-papers/2025/adrm/ces/CES-WP-25-27.pdf"
],
"source_type": "official microdata working paper on classical/industrial AI",
"evidence_family_id": "F32_CENSUS_INDUSTRIAL_AI",
"evidence_role": "Classical AI evidence, kept separate from GenAI.",
"population_context": "U.S. manufacturing establishments and firms using industrial AI/robotics.",
"method_sample": "Detailed Census firm/plant data from 2017 and 2021; analysis of adoption and performance over time.",
"verified_finding": "Short-run industrial AI adoption was associated with lower productivity and profitability, lower work-in-progress/robot investment and labor shedding; later growth among surviving adopters differed, with older firms showing losses.",
"measured_outcome": "Productivity, profitability, work-in-progress, investment and employment.",
"evidence_status": "mixed and time-dependent; some negative short-run associations",
"limits": "Working paper not endorsed as official Census findings; adoption selection and survival complicate causal interpretation; old adoption period and classical AI.",
"access_status": "Full working-paper PDF read.",
"commercial_or_promotion": "U.S. Census working paper; classical industrial AI, not GenAI."
},
{
"id": "S38",
"title": "Chicago Fed Letter on AI patents and firm performance",
"date": "2026",
"source_urls": [
"https://www.chicagofed.org/-/media/publications/chicago-fed-letter/2026/cfl518.pdf?sc_lang=en"
],
"source_type": "observational firm-level patent/capital analysis",
"evidence_family_id": "F33_CHICAGO_FED_PATENTS",
"evidence_role": "Historical AI investment proxy and firm outcomes.",
"population_context": "U.S. firms with AI-related patenting over several decades.",
"method_sample": "Regression analysis using AI patent value as a proxy for AI capital.",
"verified_finding": "A one-standard-deviation increase in AI patent value/capital (about $270,000) was associated after five years with capital +1.3%, revenue +1%, employment +0.8%, and TFP +0.3%; firm effects were stronger than sector/aggregate effects.",
"measured_outcome": "Firm capital, revenue, employment and TFP associations.",
"evidence_status": "small positive firm association; no strong aggregate inference",
"limits": "Patents proxy for AI capital; observational and older/classical AI, not direct evidence on current GenAI tools.",
"access_status": "Full PDF read.",
"commercial_or_promotion": "Federal Reserve Bank of Chicago analysis."
},
{
"id": "S39",
"title": "OECD report on measuring AI investment",
"date": "2025-09",
"source_urls": [
"https://www.oecd.org/content/dam/oecd/en/publications/reports/2025/09/advancing-the-measurement-of-investments-in-artificial-intelligence_7f58ff65/13e0da2f-en.pdf"
],
"source_type": "international statistical measurement report",
"evidence_family_id": "F34_OECD_AI_MEASUREMENT",
"evidence_role": "Measurement limitations, not an effect estimate.",
"population_context": "National accounts and AI investment measurement across OECD economies.",
"method_sample": "Review of classification and investment-statistics challenges.",
"verified_finding": "AI investment is fragmented across software, data, cloud and hardware categories and overlaps existing classifications such as robotics; official statistics incompletely capture it.",
"measured_outcome": "Measurement coverage and classification quality.",
"evidence_status": "productivity effect not measured; measurement limitation documented",
"limits": "Does not estimate AI returns or prove hidden productivity gains; descriptive measurement review.",
"access_status": "Full report PDF read.",
"commercial_or_promotion": "OECD institutional report."
},
{
"id": "S40",
"title": "FTC Operation AI Comply enforcement announcement",
"date": "2024-09",
"source_urls": [
"https://www.ftc.gov/news-events/news/press-releases/2024/09/ftc-announces-crackdown-deceptive-ai-claims-schemes"
],
"source_type": "regulatory enforcement context",
"evidence_family_id": null,
"evidence_role": "Examples of challenged commercial AI claims; not an adoption or productivity study.",
"population_context": "Specific AI products and businesses named in FTC enforcement actions.",
"method_sample": "FTC investigation and enforcement actions; allegations and orders concern named firms.",
"verified_finding": "FTC announced actions against deceptive AI-related business and income claims, including an AI lawyer marketing case whose testing claims were challenged.",
"measured_outcome": "Claim substantiation and consumer-protection compliance, not business productivity.",
"evidence_status": "contextual evidence about promotional incentives",
"limits": "Selected enforcement cases cannot estimate prevalence of misleading claims or average AI returns; allegations are specific to named cases.",
"access_status": "Full FTC release read.",
"commercial_or_promotion": "Government enforcement source."
},
{
"id": "S41",
"title": "Goldman Sachs Exchanges discussion of the AI investment boom",
"date": "2026-06-02",
"source_urls": [
"https://www.goldmansachs.com/insights/goldman-sachs-exchanges/the-ai-investment-boom-when-will-it-pay-off"
],
"source_type": "financial analyst commentary",
"evidence_family_id": null,
"evidence_role": "Contextual expert opinion, not a separate dataset.",
"population_context": "Enterprise AI buyers, model firms and hyperscalers.",
"method_sample": "Interview/commentary with Goldman Sachs analyst Jim Covello.",
"verified_finding": "Covello said enterprise buyers, model companies and hyperscalers had yet to demonstrate returns on spending; this is an attributed analyst assessment.",
"measured_outcome": "No systematic outcome measure or sample.",
"evidence_status": "contextual opinion; no empirical effect estimate",
"limits": "Analyst commentary is not independent statistical evidence; timing and definition of return are unspecified.",
"access_status": "Full article read.",
"commercial_or_promotion": "Goldman Sachs commentary."
}
]
}