{ "version": 1, "updatedAt": "2026-09-10", "description": "Reusable research protocols for testing human-AI decision patterns without treating working labels as established cognitive biases.", "canonicalUrl": "https://cognitive-biases.github.io/ai-era/protocol/", "rules": [ "Start from an observable behaviour, not a catchy label.", "State the competing explanation before collecting results.", "Change one important variable at a time where possible.", "Keep model, version, prompt, settings and date.", "Separate model behaviour from human response.", "Report null or mixed results as clearly as positive results." ], "protocols": [ { "slug": "ai-advice-order", "title": "AI Advice Order Test", "track": "ai-anchoring-loop", "question": "Does seeing an AI estimate before making your own estimate pull the final judgment toward the model?", "conditions": [ "Independent estimate first, then AI advice", "AI advice first, then human estimate" ], "primaryMeasure": "Absolute shift toward the AI estimate", "secondaryMeasures": [ "confidence change", "decision time", "distance from a reference-class estimate" ], "minimumReport": "Report the model/version, task, anchor value, sample size, mean shift by condition and uncertainty.", "preregisteredStudyId": "ai-advice-order-v1", "studyUrl": "/research/ai-advice-order-v1/", "instrumentUrl": "/experiments/ai-advice-order-v1/", "stimulusNote": "The v1 pilot isolates advice-order effects with standardized AI-labelled numerical recommendations fixed by protocol rather than live model calls." }, { "slug": "blind-vs-history-aware-ai-judging", "title": "Blind vs History-Aware AI Judging", "track": "evaluation-history-contamination", "question": "Does an earlier evaluation score change a later AI judge's score when the candidate answer and rubric stay fixed?", "conditions": [ "Blind evaluation with no history", "Prior-review framing without a score", "Prior-review framing with an experimentally assigned score and verdict" ], "primaryMeasure": "Signed movement of the history-aware score toward the displayed previous score relative to the blind score", "secondaryMeasures": [ "history-anchor versus history-framing difference", "absolute error against constructed gold score", "correction-harm rate", "verdict flip rate", "repeat dispersion" ], "minimumReport": "Use fresh contexts, keep model/version/settings fixed within a run, report every task-condition score, and publish model-level results before pooling.", "preregisteredStudyId": "ai-judge-history-v1", "studyUrl": "/research/ai-judge-history-v1/", "instrumentUrl": "/experiments/ai-judge-history-v1/", "stimulusNote": "The previous score is controlled metadata, not new evidence about candidate quality. The protocol tests evaluation sensitivity without assuming the model shares a human psychological mechanism." }, { "slug": "agreement-pressure", "title": "AI Agreement Pressure Test", "track": "sycophancy-reinforcement-loop", "question": "Does signalling a preferred answer change model agreement and then increase human confidence in that preferred answer?", "conditions": [ "Neutral question", "Question containing a preferred conclusion", "Preferred conclusion plus explicit request for counterevidence" ], "primaryMeasure": "Change in model agreement rate and user confidence", "secondaryMeasures": [ "factual accuracy", "counterargument quality", "answer revision rate" ], "minimumReport": "Keep the underlying task identical across conditions and report both model behaviour and human confidence separately." }, { "slug": "source-memory", "title": "Human-AI Source Memory Test", "track": "source-memory-blur", "question": "Do mixed human-AI workflows make it harder to remember who produced an idea or sentence?", "conditions": [ "Human-only creation", "AI-only suggestion", "Mixed human-AI creation" ], "primaryMeasure": "Correct source-attribution rate after a delay", "secondaryMeasures": [ "confidence in attribution", "verbatim recognition", "idea ownership judgment" ], "minimumReport": "Predefine the delay, keep contribution labels hidden during the test, and distinguish idea-source from wording-source errors." }, { "slug": "offloading-retention", "title": "AI Offloading & Retention Test", "track": "cognitive-offloading-debt", "question": "Can AI improve immediate completion while weakening unaided retention or transfer?", "conditions": [ "Unaided work", "AI-assisted work", "AI-assisted work plus retrieval/explanation step" ], "primaryMeasure": "Delayed unaided retention or transfer performance", "secondaryMeasures": [ "immediate task quality", "time on task", "self-rated effort", "confidence" ], "minimumReport": "Do not infer long-term learning from immediate task quality. Include a delayed or transfer measure." }, { "slug": "structured-output-authority-test", "title": "Structured Output Authority Test", "track": "structured-output-authority", "question": "Does formatting the same AI answer as a table or scorecard increase trust or reduce verification compared with equivalent prose?", "conditions": [ "Equivalent answer in plain prose", "Equivalent answer in a structured table or scorecard", "Structured answer plus a note that formatting does not increase evidence quality" ], "primaryMeasure": "Trust rating and verification rate", "secondaryMeasures": [ "error acceptance", "perceived objectivity", "decision confidence", "time to first verification action" ], "minimumReport": "Keep claims, wording, sources, model identity and planted error rate constant. Report readability separately so a fluency advantage is not silently treated as authority." }, { "slug": "dialogue-momentum-test", "title": "Dialogue Momentum Test", "track": "dialogue-momentum", "question": "After a multi-turn AI conversation converges on a plan, are people less likely to reopen an early assumption when new counterevidence appears?", "conditions": [ "Continuous conversation with prior convergence", "Fresh context containing the same plan and new counterevidence", "Continuous conversation plus an explicit instruction to reopen the original assumption" ], "primaryMeasure": "Rate of revising the challenged assumption", "secondaryMeasures": [ "final decision quality", "confidence change", "amount of counterevidence discussed", "time to revision" ], "minimumReport": "Keep the plan, counterevidence and decision task equivalent across conditions. Report whether differences come from human revision, model response, or both." }, { "slug": "citation-density-halo-test", "title": "Citation Density Halo Test", "track": "citation-density-halo", "question": "Do repeated citation markers make an AI answer look better supported when the number and quality of unique sources do not change?", "conditions": [ "Two unique sources cited once each", "The same two sources repeated through many citation markers", "Repeated markers plus explicit disclosure that only two unique sources are used" ], "primaryMeasure": "Perceived evidence strength", "secondaryMeasures": [ "trust", "verification intent", "estimate of unique source count", "error acceptance" ], "minimumReport": "Hold answer text, source quality and unique source count constant. Record whether participants correctly understand source independence rather than assuming every marker represents new evidence." } ], "startingEvidence": [ { "title": "How was my performance? Exploring the role of anchoring bias in AI-assisted decision making", "year": 2025, "url": "https://doi.org/10.1016/j.ijinfomgt.2025.102875" }, { "title": "Anchoring bias in large language models: an experimental study", "year": 2025, "url": "https://doi.org/10.1007/s42001-025-00435-2" }, { "title": "Anchoring Bias in LLM-as-a-Judge Systems: Prior Scores Compromise Evaluation Independence", "year": 2026, "url": "https://arxiv.org/abs/2608.25869" }, { "title": "Mediocrity is the key for LLM as a Judge Anchor Selection", "year": 2026, "url": "https://aclanthology.org/2026.acl-long.706/" }, { "title": "Good Arguments Against the People Pleasers: How Reasoning Mitigates (Yet Masks) LLM Sycophancy", "year": 2026, "url": "https://doi.org/10.18653/v1/2026.acl-long.1126" }, { "title": "Too Nice to Tell the Truth: Quantifying Agreeableness-Driven Sycophancy in Role-Playing Language Models", "year": 2026, "url": "https://doi.org/10.18653/v1/2026.acl-long.1421" }, { "title": "Large Language Models are overconfident and amplify human bias", "year": 2025, "url": "https://arxiv.org/abs/2505.02151" }, { "title": "The AI Memory Gap: Users Misremember What They Created With AI or Without", "year": 2025, "url": "https://arxiv.org/abs/2509.11851" }, { "title": "ChatGPT as a cognitive crutch: Evidence from a randomized controlled trial on knowledge retention", "year": 2025, "url": "https://doi.org/10.1016/j.ssaho.2025.102287" } ] }