{ "schema": "yarn-pack/2", "id": "framework-iso5259", "version": "0.1.0", "name": "ISO/IEC 5259 (framework self-assessment)", "engagement": "AISG — framework self-assessment: workshop prep, workshop, or diagnostic strand", "intro": "A guided conversation against ISO/IEC 5259, not a form. Answer in your own words and name the document or record that shows it if you can. About 20 minutes. Assesses whether the data an organisation’s AI trains on and infers from is measured, managed and governed for quality under ISO/IEC 5259. It does not assess the model itself, which 42001 and NIST cover.", "tone_default": "professional", "prefill_fields": { "department": [ "Executive", "Finance", "Operations", "Customer / Sales", "Technology / IT", "Data & Analytics", "People & Culture", "Risk & Compliance", "Marketing", "Product" ] }, "scales": [ { "id": "maturity5", "name": "Maturity (distilled model)", "levels": [ { "value": 1, "label": "Does not exist", "gloss": "No capability. Absent, or purely ad hoc / accidental." }, { "value": 2, "label": "Partially exists", "gloss": "Emerging and inconsistent. Pockets of activity, not joined up." }, { "value": 3, "label": "Fully exists", "gloss": "Defined, documented and operating across the organisation." }, { "value": 4, "label": "Fully exists & optimised", "gloss": "Measured, refined and improving against targets." }, { "value": 5, "label": "Fully exists & adaptive", "gloss": "Continuously self-adjusting; a source of advantage." } ], "signals": { "1": [ "no ", "not ", "none", "never", "don't", "do not", "nothing", "absent", "unaware", "haven't", "ad hoc", "ad-hoc", "nonexistent", "no idea", "not really" ], "2": [ "some ", "starting", "beginning", "emerging", "pilot", "trial", "informal", "inconsistent", "pockets", "a bit", "occasionally", "early", "experiment", "trying", "patchy" ], "3": [ "documented", "defined", "standard", "standardised", "established", "policy", "framework", "process", "consistent", "across the", "in place", "formal", "governed", "rolled out" ], "4": [ "measured", "metrics", "optimis", "improving", "kpi", "monitored", "reviewed", "refined", "benchmarked", "targets", "tracked", "mature", "regularly review" ], "5": [ "continuous", "adaptive", "self-", "automated end", "best in class", "best-in-class", "competitive advantage", "industry leading", "always", "real-time monitoring", "feedback loop" ] } } ], "categories": [ { "id": "characteristics", "name": "Data quality characteristics", "order": 1, "target_default": 3 }, { "id": "process", "name": "Quality process and lineage", "order": 2, "target_default": 3 } ], "audiences": [ { "id": "lead", "name": "Governance / risk lead", "desc": "Owns the policy, the register or the risk framework", "deep_dive_sections": [] }, { "id": "owner", "name": "System or use-case owner", "desc": "Runs an AI system or use case day to day", "deep_dive_sections": [] }, { "id": "exec", "name": "Executive / sponsor", "desc": "Accountable for the outcome, not the mechanics", "deep_dive_sections": [] } ], "sections": [ { "id": "core", "title": "5259 elements", "blurb": "Everyone answers these. The data team (Part 2 measures), the data quality lead (Part 3), the engineers who run the pipelines (Part 4), and the CDO or board delegate who oversees data quality (Part 5).", "optional": false, "questions": [ { "id": "5259-1", "type": "scored_text", "category": "characteristics", "name": "Accuracy and completeness", "text": "When you last trained or refreshed a model, how did you know the data was right and nothing required was missing? Can you show me the number?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No measure of accuracy or completeness exists for any ML dataset; fitness for AI is asserted from general belief about the data.", "3": "Each ML dataset has a baselined accuracy and completeness measure, recorded with the dataset, that an auditor can read.", "5": "Accuracy and completeness measures run on each refresh, and a drop below threshold blocks or flags the dataset without anyone prompting it." }, "help": "Evidence that would show it: Baseline data quality measures per ML dataset (Part 2); Model card recording accuracy and completeness of training data; Monitoring output showing the measures over time.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-2", "type": "scored_text", "category": "characteristics", "name": "Consistency and currency", "text": "How old can the data behind this AI be before it is wrong for the job? And when two sources disagree, which one does the model believe, and who decided that?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No check for contradictions between sources and no stated currency requirement; stale or conflicting data reaches the model unnoticed.", "3": "Consistency checks and a currency requirement are defined per ML dataset, measured at baseline, and recorded with the dataset.", "5": "Contradictions and staleness are detected on each refresh, and corrections propagate to every system grounded on the data without manual chasing." }, "help": "Evidence that would show it: Consistency and currency measures recorded per dataset; Refresh cadence stated for each corpus or dataset; Record of a correction reaching the systems that use the data.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-3", "type": "scored_text", "category": "characteristics", "name": "Representativeness", "text": "Who does this AI make decisions about, and how do you know the data it learnt from looks like those people? What would you show me?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Nobody has compared the training data to the population the AI serves; representativeness is assumed or has never been raised.", "3": "Each ML dataset has a documented representativeness measure against a defined real-world population, baselined before training and recorded.", "5": "Representativeness is re-measured on every refresh and after deployment, and drift from the population triggers review without anyone asking." }, "help": "Evidence that would show it: Representativeness measure against a defined population, per dataset; Model card stating the population the data reflects and its limits; Fairness risk entry that cites the data-side measure.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-4", "type": "scored_text", "category": "characteristics", "name": "Balance", "text": "If one type of case is rare in your data, what did you do about it before training? How did you decide the mix was right for what the model has to learn?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No one has examined the distribution of classes or cases in the training data; whatever came out of the source is what the model learnt from.", "3": "The distribution of each training dataset is measured against the learning task and recorded, with the fit decided before training proceeds.", "5": "Balance is re-measured on every refresh, and the pipeline re-weights, re-samples or halts when the distribution moves away from the task." }, "help": "Evidence that would show it: Distribution measure per training dataset, recorded against the task; Stage-gate criterion requiring balance to be assessed before training; Model card recording the distribution and any re-sampling.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-5", "type": "scored_text", "category": "characteristics", "name": "Availability and accessibility", "text": "When this AI needs data, where does it get it from, and what happens if that source is not there? Has anyone tested that it can reach what it needs, and nothing more?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Data reaches the AI by whatever path was convenient; nobody can say which stores it can reach, or whether it gets the data when it needs it.", "3": "For each AI system the data it needs, the store it comes from and the access path are documented, and access is tested rather than asserted.", "5": "Availability is monitored continuously; a source that becomes unreachable or changes is detected and the affected AI is flagged before it answers wrongly." }, "help": "Evidence that would show it: Documented data sources and access path per AI system; Tested data-access controls on every store an AI can reach; Corpus register with owner and refresh cadence.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-6", "type": "scored_text", "category": "process", "name": "Three-type data quality model", "text": "If I asked three people here what “good data for AI” means, would they name the same things? Where is that list written down, and does it include anything specific to machine learning?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Data quality is spoken of in general terms; there is no shared set of characteristics, and ML-specific quality is not a category anyone uses.", "3": "One characteristic set, including the ML-specific type, is written into data quality procedures and used by the data team, the AI team and risk alike.", "5": "The same characteristic set is what DMBOK data quality, 42001 Annex A.7 and EU AI Act evidence all cite, so one measurement serves every claim." }, "help": "Evidence that would show it: Data quality procedure listing inherent, system-dependent and ML-specific characteristics; Intake form capturing data quality against the characteristic set; Controls library entry that names the characteristic set.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-7", "type": "scored_text", "category": "process", "name": "Traceability and provenance", "text": "Pick one model. Can you walk me back from it to where every piece of its training data came from, and what changed along the way? Who would know if the source changed?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Nobody can say where a training dataset came from or what was done to it; provenance lives in someone’s memory, if anywhere.", "3": "Each ML dataset has a recorded origin, owner and transformation history that an auditor can follow from source to model without asking the engineer.", "5": "Lineage is captured by the pipeline as it runs, and a change at any upstream source or vendor is traced to every model built on it." }, "help": "Evidence that would show it: Lineage record per dataset: origin, owner, transformations; Corpus register naming who owns the content, not the pipeline; Boundary declaration for third-party data: whose data, which agreement; Vendor material change notice covering training data changes.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-8", "type": "scored_text", "category": "process", "name": "Data quality management system and process", "text": "Who owns data quality for AI here, day to day? Walk me through what happens to a dataset from the moment it is collected to when it is retired: where is quality checked?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "No DQMS exists and no defined data quality process runs across the lifecycle; data quality work, where it happens, is ad hoc and undocumented.", "3": "A DQMS with a named lead, and lifecycle procedures with a quality step at each stage from collection to retirement, exist and are in use for ML data.", "5": "The DQMS improves itself: measures feed back into procedure changes, and the lifecycle steps adjust as new data sources and model types arrive." }, "help": "Evidence that would show it: DQMS documentation with a named data quality lead; Lifecycle SOPs with a data quality step at each stage; Stage-gate criterion requiring data quality measures before build.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-9", "type": "scored_text", "category": "process", "name": "Data quality governance", "text": "Who above the data team would be held to account if an AI went wrong because of bad data? When did they last see a data quality measure, and what did they do with it?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Data quality for AI has no oversight above the team that produces it; the board has never seen a data quality measure for any AI system.", "3": "A named executive owns data quality for AI, it has a place on a standing governance body, and measures reach the board on a stated cadence.", "5": "Board reporting on data quality is generated from live measures, and the governance body changes thresholds or priorities in response to what it sees." }, "help": "Evidence that would show it: Board or audit committee report containing data quality measures; Governance body charter naming data quality for AI in scope; Named executive owner for data quality risk; RACI showing data quality decision rights across the lifecycle.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "5259-10", "type": "scored_text", "category": "process", "name": "Recurring measurement and drift", "text": "When did you last measure the quality of the data this AI is using, not when it was built, but last month? What would tell you it had drifted, and who would hear?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Data quality was assessed once, in the past tense, and never re-run; nobody would notice if the data drifted and the model went quietly wrong.", "3": "A recurring data quality check runs on a stated schedule for each ML dataset, evidence of each run is kept, and drift triggers a defined response.", "5": "Measures run continuously in monitoring, drift is acted on before model output degrades, and the check schedule adapts to how fast each source moves." }, "help": "Evidence that would show it: Dated evidence of repeated data quality runs, per dataset; Assurance schedule listing the recurring data quality check; Monitoring output showing drift detection and the response taken.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false } ] } ], "grids": {}, "outputs": [ "Level per element and per category, gated", "Contested-element view (spread of 2 or more)", "Coverage of evidence: confirmed, stated, inferred", "Where to start, foundations first", "Printable client report" ], "report_defaults": [ "rpt-maturity-standard" ], "playbook": { "sequence": [ "characteristics", "process" ], "sequence_note": "Adopt the characteristic set as vocabulary, baseline current ML datasets against it, then stand up the DQMS, process and governance, and wire the measures into monitoring.", "actions": { "characteristics": { "to_3": [ "Adopt the 5259 characteristic set as the common data quality vocabulary", "Baseline every current ML dataset against the Part 2 measures and record the result" ], "to_5": [ "Wire the characteristic measures into monitoring so they run on every refresh, not once" ] }, "process": { "to_3": [ "Stand up a DQMS (Part 3) with a named lead and a lifecycle process (Part 4)", "Put governance over it (Part 5) tied to the board", "Record origin, owner and transformations for each ML dataset" ], "to_5": [ "Build the recurring data quality check with evidence of each run", "Treat “we assessed data quality” in the past tense as a warning sign and re-run" ] } } } }