diff --git a/preregistration-consistency-assistant/README.md b/preregistration-consistency-assistant/README.md new file mode 100644 index 00000000..14a90a0a --- /dev/null +++ b/preregistration-consistency-assistant/README.md @@ -0,0 +1,25 @@ +# Preregistration consistency assistant + +This module checks whether AI review output should be released when a study has a preregistration plan. + +It focuses on one AI research assistant slice: spotting preregistration conflicts before the assistant gives authors reviewer-facing guidance. It does not call external registries, run AI models, or process private manuscripts. + +## What it checks + +- preregistration id +- primary outcome reporting +- unlogged primary outcome changes +- unexplained analysis plan changes +- unplanned subgroup claims +- sample size shortfalls +- secondary outcome promotion +- missing null result discussion + +## Run it + +```bash +node preregistration-consistency-assistant/test.js +node preregistration-consistency-assistant/demo.js +``` + +The demo writes JSON and Markdown artifacts to `artifacts/`. diff --git a/preregistration-consistency-assistant/artifacts/demo.gif b/preregistration-consistency-assistant/artifacts/demo.gif new file mode 100644 index 00000000..511d360d Binary files /dev/null and b/preregistration-consistency-assistant/artifacts/demo.gif differ diff --git a/preregistration-consistency-assistant/artifacts/demo.mp4 b/preregistration-consistency-assistant/artifacts/demo.mp4 new file mode 100644 index 00000000..d4fcdebd Binary files /dev/null and b/preregistration-consistency-assistant/artifacts/demo.mp4 differ diff --git a/preregistration-consistency-assistant/artifacts/preregistration-report.md b/preregistration-consistency-assistant/artifacts/preregistration-report.md new file mode 100644 index 00000000..914db41b --- /dev/null +++ b/preregistration-consistency-assistant/artifacts/preregistration-report.md @@ -0,0 +1,27 @@ +# Preregistration consistency assistant report + +## STUDY-01: Sleep intervention trial +Decision: RELEASE_ASSISTANT_REVIEW +Next step: release the assistant review with preregistration receipt + +## STUDY-02: Microbiome cohort analysis +Decision: FLAG_FOR_AUTHOR_CHECK +Next step: ask authors to address warnings before final release +Warnings: +- sample size is below the preregistered plan +- secondary outcomes are promoted above the primary outcome +- null results are not mentioned + +## STUDY-03: Cognitive training trial +Decision: HOLD_ASSISTANT_REVIEW +Next step: hold AI review output until preregistration conflicts are resolved +Blockers: +- missing preregistration id +- primary outcome is not reported +- primary outcome changed without amendment +- analysis plan changed without explanation +- unplanned subgroup claim is not marked exploratory +Warnings: +- sample size is below the preregistered plan +- secondary outcomes are promoted above the primary outcome +- null results are not mentioned diff --git a/preregistration-consistency-assistant/artifacts/preregistration-results.json b/preregistration-consistency-assistant/artifacts/preregistration-results.json new file mode 100644 index 00000000..ac4e0904 --- /dev/null +++ b/preregistration-consistency-assistant/artifacts/preregistration-results.json @@ -0,0 +1,40 @@ +[ + { + "studyId": "STUDY-01", + "title": "Sleep intervention trial", + "decision": "RELEASE_ASSISTANT_REVIEW", + "blockers": [], + "warnings": [], + "nextStep": "release the assistant review with preregistration receipt" + }, + { + "studyId": "STUDY-02", + "title": "Microbiome cohort analysis", + "decision": "FLAG_FOR_AUTHOR_CHECK", + "blockers": [], + "warnings": [ + "sample size is below the preregistered plan", + "secondary outcomes are promoted above the primary outcome", + "null results are not mentioned" + ], + "nextStep": "ask authors to address warnings before final release" + }, + { + "studyId": "STUDY-03", + "title": "Cognitive training trial", + "decision": "HOLD_ASSISTANT_REVIEW", + "blockers": [ + "missing preregistration id", + "primary outcome is not reported", + "primary outcome changed without amendment", + "analysis plan changed without explanation", + "unplanned subgroup claim is not marked exploratory" + ], + "warnings": [ + "sample size is below the preregistered plan", + "secondary outcomes are promoted above the primary outcome", + "null results are not mentioned" + ], + "nextStep": "hold AI review output until preregistration conflicts are resolved" + } +] \ No newline at end of file diff --git a/preregistration-consistency-assistant/demo.js b/preregistration-consistency-assistant/demo.js new file mode 100644 index 00000000..8a20072b --- /dev/null +++ b/preregistration-consistency-assistant/demo.js @@ -0,0 +1,13 @@ +const fs = require("fs"); +const path = require("path"); +const studies = require("./sample-data.json"); +const { evaluateStudies, renderMarkdownReport } = require("./index"); + +const artifactsDir = path.join(__dirname, "artifacts"); +fs.mkdirSync(artifactsDir, { recursive: true }); + +const results = evaluateStudies(studies); +fs.writeFileSync(path.join(artifactsDir, "preregistration-results.json"), JSON.stringify(results, null, 2)); +fs.writeFileSync(path.join(artifactsDir, "preregistration-report.md"), renderMarkdownReport(results)); + +console.log(renderMarkdownReport(results)); diff --git a/preregistration-consistency-assistant/index.js b/preregistration-consistency-assistant/index.js new file mode 100644 index 00000000..938d2c38 --- /dev/null +++ b/preregistration-consistency-assistant/index.js @@ -0,0 +1,64 @@ +function evaluateStudy(study) { + const blockers = []; + const warnings = []; + + if (!study.preregistrationId) blockers.push("missing preregistration id"); + if (!study.primaryOutcomeReported) blockers.push("primary outcome is not reported"); + if (study.primaryOutcomeChanged && !study.amendmentLogged) blockers.push("primary outcome changed without amendment"); + if (study.analysisPlanChanged && !study.deviationExplained) blockers.push("analysis plan changed without explanation"); + if (study.unplannedSubgroupClaim && !study.markedExploratory) blockers.push("unplanned subgroup claim is not marked exploratory"); + if (study.sampleSize < study.plannedSampleSize) warnings.push("sample size is below the preregistered plan"); + if (study.secondaryOutcomesPromoted) warnings.push("secondary outcomes are promoted above the primary outcome"); + if (!study.nullResultsMentioned) warnings.push("null results are not mentioned"); + + let decision = "RELEASE_ASSISTANT_REVIEW"; + if (blockers.length) { + decision = "HOLD_ASSISTANT_REVIEW"; + } else if (warnings.length) { + decision = "FLAG_FOR_AUTHOR_CHECK"; + } + + return { + studyId: study.id, + title: study.title, + decision, + blockers, + warnings, + nextStep: nextStepFor(decision), + }; +} + +function nextStepFor(decision) { + if (decision === "RELEASE_ASSISTANT_REVIEW") return "release the assistant review with preregistration receipt"; + if (decision === "FLAG_FOR_AUTHOR_CHECK") return "ask authors to address warnings before final release"; + return "hold AI review output until preregistration conflicts are resolved"; +} + +function evaluateStudies(studies) { + return studies.map(evaluateStudy); +} + +function renderMarkdownReport(results) { + const lines = ["# Preregistration consistency assistant report", ""]; + for (const result of results) { + lines.push(`## ${result.studyId}: ${result.title}`); + lines.push(`Decision: ${result.decision}`); + lines.push(`Next step: ${result.nextStep}`); + if (result.blockers.length) { + lines.push("Blockers:"); + for (const blocker of result.blockers) lines.push(`- ${blocker}`); + } + if (result.warnings.length) { + lines.push("Warnings:"); + for (const warning of result.warnings) lines.push(`- ${warning}`); + } + lines.push(""); + } + return lines.join("\n"); +} + +module.exports = { + evaluateStudies, + evaluateStudy, + renderMarkdownReport, +}; diff --git a/preregistration-consistency-assistant/sample-data.json b/preregistration-consistency-assistant/sample-data.json new file mode 100644 index 00000000..96e27a41 --- /dev/null +++ b/preregistration-consistency-assistant/sample-data.json @@ -0,0 +1,50 @@ +[ + { + "id": "STUDY-01", + "title": "Sleep intervention trial", + "preregistrationId": "OSF-123", + "primaryOutcomeReported": true, + "primaryOutcomeChanged": false, + "amendmentLogged": false, + "analysisPlanChanged": false, + "deviationExplained": false, + "unplannedSubgroupClaim": false, + "markedExploratory": false, + "sampleSize": 240, + "plannedSampleSize": 220, + "secondaryOutcomesPromoted": false, + "nullResultsMentioned": true + }, + { + "id": "STUDY-02", + "title": "Microbiome cohort analysis", + "preregistrationId": "OSF-778", + "primaryOutcomeReported": true, + "primaryOutcomeChanged": false, + "amendmentLogged": false, + "analysisPlanChanged": false, + "deviationExplained": false, + "unplannedSubgroupClaim": false, + "markedExploratory": false, + "sampleSize": 141, + "plannedSampleSize": 180, + "secondaryOutcomesPromoted": true, + "nullResultsMentioned": false + }, + { + "id": "STUDY-03", + "title": "Cognitive training trial", + "preregistrationId": "", + "primaryOutcomeReported": false, + "primaryOutcomeChanged": true, + "amendmentLogged": false, + "analysisPlanChanged": true, + "deviationExplained": false, + "unplannedSubgroupClaim": true, + "markedExploratory": false, + "sampleSize": 86, + "plannedSampleSize": 120, + "secondaryOutcomesPromoted": true, + "nullResultsMentioned": false + } +] diff --git a/preregistration-consistency-assistant/test.js b/preregistration-consistency-assistant/test.js new file mode 100644 index 00000000..4f7b8c55 --- /dev/null +++ b/preregistration-consistency-assistant/test.js @@ -0,0 +1,15 @@ +const assert = require("assert"); +const studies = require("./sample-data.json"); +const { evaluateStudies } = require("./index"); + +const results = evaluateStudies(studies); +const byId = Object.fromEntries(results.map((result) => [result.studyId, result])); + +assert.strictEqual(byId["STUDY-01"].decision, "RELEASE_ASSISTANT_REVIEW"); +assert.strictEqual(byId["STUDY-02"].decision, "FLAG_FOR_AUTHOR_CHECK"); +assert.ok(byId["STUDY-02"].warnings.some((warning) => warning.includes("sample size"))); +assert.strictEqual(byId["STUDY-03"].decision, "HOLD_ASSISTANT_REVIEW"); +assert.ok(byId["STUDY-03"].blockers.some((blocker) => blocker.includes("preregistration"))); +assert.ok(byId["STUDY-03"].blockers.some((blocker) => blocker.includes("subgroup"))); + +console.log("preregistration consistency assistant tests passed");