{ "schema": "yarn-pack/2", "id": "framework-iso23894", "version": "0.1.0", "name": "ISO/IEC 23894 (framework self-assessment)", "engagement": "AISG — framework self-assessment: workshop prep, workshop, or diagnostic strand", "intro": "A guided conversation against ISO/IEC 23894, not a form. Answer in your own words and name the document or record that shows it if you can. About 20 minutes. Assesses whether the organisation runs the ISO 31000 risk cycle on its AI, and whether the AI-specific risk sources are identified and treated. Not a certification check or a controls audit: 23894 prescribes no controls.", "tone_default": "professional", "prefill_fields": { "department": [ "Executive", "Finance", "Operations", "Customer / Sales", "Technology / IT", "Data & Analytics", "People & Culture", "Risk & Compliance", "Marketing", "Product" ] }, "scales": [ { "id": "maturity5", "name": "Maturity (distilled model)", "levels": [ { "value": 1, "label": "Does not exist", "gloss": "No capability. Absent, or purely ad hoc / accidental." }, { "value": 2, "label": "Partially exists", "gloss": "Emerging and inconsistent. Pockets of activity, not joined up." }, { "value": 3, "label": "Fully exists", "gloss": "Defined, documented and operating across the organisation." }, { "value": 4, "label": "Fully exists & optimised", "gloss": "Measured, refined and improving against targets." }, { "value": 5, "label": "Fully exists & adaptive", "gloss": "Continuously self-adjusting; a source of advantage." } ], "signals": { "1": [ "no ", "not ", "none", "never", "don't", "do not", "nothing", "absent", "unaware", "haven't", "ad hoc", "ad-hoc", "nonexistent", "no idea", "not really" ], "2": [ "some ", "starting", "beginning", "emerging", "pilot", "trial", "informal", "inconsistent", "pockets", "a bit", "occasionally", "early", "experiment", "trying", "patchy" ], "3": [ "documented", "defined", "standard", "standardised", "established", "policy", "framework", "process", "consistent", "across the", "in place", "formal", "governed", "rolled out" ], "4": [ "measured", "metrics", "optimis", "improving", "kpi", "monitored", "reviewed", "refined", "benchmarked", "targets", "tracked", "mature", "regularly review" ], "5": [ "continuous", "adaptive", "self-", "automated end", "best in class", "best-in-class", "competitive advantage", "industry leading", "always", "real-time monitoring", "feedback loop" ] } } ], "categories": [ { "id": "process", "name": "The AI-applied risk process", "order": 1, "target_default": 3 }, { "id": "sources", "name": "AI-specific risk sources", "order": 2, "target_default": 3 } ], "audiences": [ { "id": "lead", "name": "Governance / risk lead", "desc": "Owns the policy, the register or the risk framework", "deep_dive_sections": [] }, { "id": "owner", "name": "System or use-case owner", "desc": "Runs an AI system or use case day to day", "deep_dive_sections": [] }, { "id": "exec", "name": "Executive / sponsor", "desc": "Accountable for the outcome, not the mechanics", "deep_dive_sections": [] } ], "sections": [ { "id": "core", "title": "23894 elements", "blurb": "Everyone answers these. The enterprise risk owner (whoever runs the ISO 31000 process), the AI or data lead, and whoever keeps the AI risk register. Add the 42001 lead if certification is the goal.", "optional": false, "questions": [ { "id": "p-consult", "type": "scored_text", "category": "process", "name": "Consultation, scope, context and criteria", "text": "When you last looked at the risks of an AI system, who did you talk to outside the project, and what had you decided counted as too risky before you started?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No one is consulted beyond the project team; there is no stated scope, context or criteria for AI risk, so each assessment sets its own bar.", "3": "Each AI risk assessment records who was consulted, its scope and context, and the criteria used, and those criteria sit inside the enterprise risk framework.", "5": "The stakeholder list and criteria are revisited as use changes; newly affected groups are added without a prompt, and criteria updates reach every open assessment." }, "help": "Evidence that would show it: Stakeholder list per AI risk assessment, with a consultation record; Scope, context and criteria section in the AI risk method or register; Enterprise risk criteria showing AI risk within them, not beside them.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "p-identify", "type": "scored_text", "category": "process", "name": "Risk identification", "text": "Take one AI system you run. What could go wrong with it that could not go wrong with ordinary software, and where is that written down?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No AI-specific risk identification; AI systems go through the generic IT or project risk checklist, or through none.", "3": "Register entries per AI system name bias, opacity, drift, autonomy and data dependence where they apply, each dated and tied to the system it belongs to.", "5": "Identification reruns on its own when a system, its data or its use changes; a new risk type found in one system is added to the library for all of them." }, "help": "Evidence that would show it: AI risk register entries per system; Risk identification checklist naming the AI-specific risk types; Intake record showing risk indicators captured at the start.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "p-analyse", "type": "scored_text", "category": "process", "name": "Risk analysis and evaluation", "text": "For a risk you rated on an AI system, how did you rate it when the same input can give a different answer tomorrow, and who decided it was acceptable?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Risks are listed but never rated, or the ratings assume the system behaves the same way each time it runs.", "3": "Each register entry carries an analysis that records how variable outputs were handled, a rating against the criteria, and a treat or accept decision.", "5": "Analysis draws on the observed behaviour of the running system, ratings move when that behaviour moves, and the criteria are revised from what evaluation shows." }, "help": "Evidence that would show it: Rated entries in the AI risk register with the method shown; Risk analysis method noting how non-deterministic behaviour is assessed; Evaluation decisions recorded against the risk criteria.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "p-treat", "type": "scored_text", "category": "process", "name": "Risk treatment and residual risk", "text": "Pick one AI risk you have done something about. What did you do, what risk is left over now, and who signed off that the leftover is fine?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Risks are rated but no treatment is chosen, or treatments are listed with no record of what risk remains after them.", "3": "Each treated risk in the register shows the treatment, who applied it, and the residual rating, with the residual accepted by a named person.", "5": "Residual risk is re-rated as treatments are tested in operation; a treatment that stops working is flagged and replaced without waiting for the next review." }, "help": "Evidence that would show it: Treatment plan per risk, with an owner; Residual risk rating and acceptance record; Controls applied per risk, with verification.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "p-monitor", "type": "scored_text", "category": "process", "name": "Monitoring and review", "text": "When did you last go back and look at the risks of an AI system that has been live for a while, and what made you look: a date, an incident, or a change?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "One risk assessment at go-live, then nothing: no review dates, no monitoring, and the register is not opened again.", "3": "Each AI system has a monitoring routine and a review cadence, both run to date, and reviews are recorded with the changes they caused in the register.", "5": "Monitoring output feeds the register directly; drift, incidents and changes in use trigger a review on their own rather than on the calendar." }, "help": "Evidence that would show it: Monitoring outputs per AI system, dated; Review schedule with the completed reviews; Register history showing changes made after review; Incident register entries tied back to risks.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "p-record", "type": "scored_text", "category": "process", "name": "Recording and reporting", "text": "If an auditor asked tomorrow to see how you decided an AI system was safe enough to run, what would you hand over, and how long would it take to find?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Risk decisions live in email, chat or memory; nothing can be produced that shows what was decided about an AI system and by whom.", "3": "Records exist for each step of the process per AI system, and a report on AI risk reaches the accountable body on a stated cadence.", "5": "Reports are produced from the working register rather than assembled for the audit, and the accountable body’s questions change what gets recorded next." }, "help": "Evidence that would show it: Board or committee report on AI risk, with the cadence shown; Verification log or evidence register; Audit trail on the AI risk register.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "s-data", "type": "scored_text", "category": "sources", "name": "Data-related risk sources", "text": "For one AI system, where does its data come from, how do you know it is still good, and what would tell you if it had quietly changed?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Data behind AI systems is not assessed for quality, bias, drift or provenance; the risk register has no data-related entries.", "3": "Each AI system’s register entries cover data quality, bias, drift and provenance, with treatment recorded and the data sources named.", "5": "Data quality and drift are measured in operation and the measurements update the register; a provenance change on a source reopens the risk on its own." }, "help": "Evidence that would show it: Data-related entries in the AI risk register; Data quality or provenance records for each AI system’s data; Corpus or data source register with ownership.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "s-model", "type": "scored_text", "category": "sources", "name": "Model-related risk sources", "text": "Can you explain to the person affected why this system gave the answer it gave, and what happens to that answer when the input is slightly unusual?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Models are taken as given; no one has written down what the model cannot explain, where it may overfit, or how it behaves under variation.", "3": "Register entries per AI system cover opacity, explainability, overfitting, non-determinism and robustness, each with a treatment and a model record behind it.", "5": "Model behaviour is tested against these sources on a running basis, and a change of model version reopens each entry without a manual prompt." }, "help": "Evidence that would show it: Model card or equivalent per use case, with limitations stated; Model-related entries in the AI risk register; Test or evaluation results for robustness and explainability.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "s-use", "type": "scored_text", "category": "sources", "name": "Use and context risk sources", "text": "Who is meant to check this system’s output before it counts, do they actually do it, and is the system now being used for anything it was not set up for?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Intended use is not written down, so misuse and scope creep cannot be seen, and there is no human oversight model to fail.", "3": "Each AI system has a stated intended use, a documented human oversight model, and register entries for misuse, automation bias, scope creep and oversight failure.", "5": "Actual use is compared with intended use in operation; a drift in use, or a sign that people defer to the system unchecked, triggers a review." }, "help": "Evidence that would show it: Human oversight model per use case, with rationale; Acceptable use guidelines; Register entries for misuse and automation bias; Use case register with intended use stated.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "s-supply", "type": "scored_text", "category": "sources", "name": "Lifecycle and supply chain risk sources", "text": "Which of your AI comes from a vendor, how would you find out if they changed the model underneath you, and what happens to the system when you stop using it?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Third-party models are used without a register, vendor changes arrive unannounced, and no AI system has an end-of-life plan.", "3": "A vendor AI register exists, contracts require material change notification, environment change sits in the register, and decommissioning is a documented step.", "5": "Vendor changes and environment shifts flow into the register as they happen, and retirement of a system closes its risks with the records kept." }, "help": "Evidence that would show it: Vendor and third-party AI register; Material change notification process and contract clauses; Decommissioning checklist with completed examples.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false } ] } ], "grids": {}, "outputs": [ "Level per element and per category, gated", "Contested-element view (spread of 2 or more)", "Coverage of evidence: confirmed, stated, inferred", "Where to start, foundations first", "Printable client report" ], "report_defaults": [ "rpt-maturity-standard" ], "playbook": { "sequence": [ "process", "sources" ], "sequence_note": "Overlay onto the existing risk framework; do not build a parallel AI risk silo. Pick one risk method: 23894 for ISO 31000 shops, NIST AI RMF for US or voluntary leaners.", "actions": { "process": { "to_3": [ "Run the full 31000 cycle on each AI system and record every step in the enterprise register", "Set a review cadence and a monitoring routine for every live AI system", "Report AI risk to the accountable body on a stated cadence" ], "to_5": [ "Wire monitoring output and incidents into the register so that review triggers itself" ] }, "sources": { "to_3": [ "Seed identification with the four source families as a starter library", "Plug ISO/IEC 5259 data-risk evidence into identification" ], "to_5": [ "Pair the library with the NIST trustworthy-AI characteristics and the EU AI Act risk view: same risks, three vocabularies" ] } } } }