{ "schema": "yarn-pack/2", "id": "framework-agt", "version": "0.1.0", "name": "Microsoft Agent Governance Toolkit (framework self-assessment)", "engagement": "AISG — framework self-assessment: workshop prep, workshop, or diagnostic strand", "intro": "A guided conversation against Microsoft Agent Governance Toolkit, not a form. Answer in your own words and name the document or record that shows it if you can. About 18 minutes. Readiness to run a runtime governance layer for AI agents, by any means: a verdict before each action, agent identity, a tool gateway, a kill switch and tamper-evident audit. Not the management system or the corpus.", "tone_default": "professional", "prefill_fields": { "department": [ "Executive", "Finance", "Operations", "Customer / Sales", "Technology / IT", "Data & Analytics", "People & Culture", "Risk & Compliance", "Marketing", "Product" ] }, "scales": [ { "id": "maturity5", "name": "Maturity (distilled model)", "levels": [ { "value": 1, "label": "Does not exist", "gloss": "No capability. Absent, or purely ad hoc / accidental." }, { "value": 2, "label": "Partially exists", "gloss": "Emerging and inconsistent. Pockets of activity, not joined up." }, { "value": 3, "label": "Fully exists", "gloss": "Defined, documented and operating across the organisation." }, { "value": 4, "label": "Fully exists & optimised", "gloss": "Measured, refined and improving against targets." }, { "value": 5, "label": "Fully exists & adaptive", "gloss": "Continuously self-adjusting; a source of advantage." } ], "signals": { "1": [ "no ", "not ", "none", "never", "don't", "do not", "nothing", "absent", "unaware", "haven't", "ad hoc", "ad-hoc", "nonexistent", "no idea", "not really" ], "2": [ "some ", "starting", "beginning", "emerging", "pilot", "trial", "informal", "inconsistent", "pockets", "a bit", "occasionally", "early", "experiment", "trying", "patchy" ], "3": [ "documented", "defined", "standard", "standardised", "established", "policy", "framework", "process", "consistent", "across the", "in place", "formal", "governed", "rolled out" ], "4": [ "measured", "metrics", "optimis", "improving", "kpi", "monitored", "reviewed", "refined", "benchmarked", "targets", "tracked", "mature", "regularly review" ], "5": [ "continuous", "adaptive", "self-", "automated end", "best in class", "best-in-class", "competitive advantage", "industry leading", "always", "real-time monitoring", "feedback loop" ] } } ], "categories": [ { "id": "policy", "name": "Policy decision before execution", "order": 1, "target_default": 3 }, { "id": "identity", "name": "Agent identity, tools and containment", "order": 2, "target_default": 3 }, { "id": "evidence", "name": "Enforcement surfaces and evidence", "order": 3, "target_default": 3 } ], "audiences": [ { "id": "lead", "name": "Governance / risk lead", "desc": "Owns the policy, the register or the risk framework", "deep_dive_sections": [] }, { "id": "owner", "name": "System or use-case owner", "desc": "Runs an AI system or use case day to day", "deep_dive_sections": [] }, { "id": "exec", "name": "Executive / sponsor", "desc": "Accountable for the outcome, not the mechanics", "deep_dive_sections": [] } ], "sections": [ { "id": "core", "title": "MS AGT elements", "blurb": "Everyone answers these. The platform lead who owns the API gateway, the identity administrator, the security lead or vCISO, each agent’s human sponsor, and the governance lead who owns the register and controls library.", "optional": false, "questions": [ { "id": "agt-1", "type": "scored_text", "category": "policy", "name": "Policy verdict before every action", "text": "When one of your agents is about to call a tool or send something out, what happens in the moment before it does? Who or what can stop it, and could you show me where that rule is written down?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Agents call tools directly. The only limits are OAuth scopes, IAM roles and instructions in the prompt; nobody can name a policy file that governs what an agent does.", "3": "A versioned policy manifest is evaluated before each tool call and output; the host blocks on deny, and a log shows verdicts with policy IDs and reason codes per action.", "5": "Verdict counts and deny rates feed a periodic review that tightens or relaxes rules; one manifest is enforced unchanged across every framework, gateway and custom host." }, "help": "Evidence that would show it: Policy manifest (Rego, Cedar or YAML) under version control; Verdict log with policy IDs and reason codes per action; Design showing the enforcement point sits between the agent and its tools.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-2", "type": "scored_text", "category": "policy", "name": "Fail closed and no empty policy set", "text": "If the thing that checks your agents’ actions crashed or timed out this afternoon, would the agents stop or carry on? And how would you know today if one of them had started up with no rules loaded at all?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Nobody knows what happens when the policy check errors or times out. Hosts can start with no policy loaded and nothing reports that they did.", "3": "Errors, timeouts and unconfigured approvals are proven to block. A health check such as agt doctor runs green on every governed host; no production host is permissive.", "5": "The health check runs on every deploy and on a schedule; a host with an empty policy set or permissive mode is pulled before it serves traffic; the finding is logged." }, "help": "Evidence that would show it: agt doctor or equivalent health-check output per governed host; Test record showing a forced evaluation error returned deny; Controls library line: no host runs with an empty policy set.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-3", "type": "scored_text", "category": "policy", "name": "Approval checkpoints the agent cannot shape", "text": "Which things do your agents have to ask a human before doing? When they ask, what does that person actually see on their screen, and who wrote what they see?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Approvals, where they exist, are the agent asking in its own words inside the chat; nobody can list which actions require a person or who that person is.", "3": "A written list names which actions escalate and to whom; approvals are recorded against the exact action; the approver sees a view rendered by the host, not agent text.", "5": "Escalation rules are tuned from approval outcomes; approvals made under time pressure are sampled and reviewed; rules that never escalate or always approve are retired." }, "help": "Evidence that would show it: Escalation map: action types, approval backend, named approvers; Approval records bound to the action they authorised; Specification or screenshot of the approver view showing host-rendered content.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-4", "type": "scored_text", "category": "identity", "name": "Distinct identity with a named human sponsor", "text": "If I picked one of your agents at random, could you tell me its name in the directory, who in the business answers for it, and what happens if that person leaves? Has anyone ever switched one off from the directory?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Agents run under a shared service account or a staff member’s own login; no register lists the agents, and nobody is named as responsible for any of them.", "3": "Every agent has its own directory identity, a named human sponsor and a register row; credentials are short-lived; a test shows suspending the identity stops the agent.", "5": "An agent cannot be created without a sponsor; shadow-agent discovery adds unregistered agents to the register on its own; access reviews expire agents whose sponsor left." }, "help": "Evidence that would show it: Agent register with identity, sponsor and lifecycle state per row; Directory entries for agents with Conditional Access and access reviews applied; Test record of a suspension stopping a running agent.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-5", "type": "scored_text", "category": "identity", "name": "Tool gateway for every MCP call", "text": "Walk me through what happens between your agent asking for a tool and the tool answering. Who decides which tools it may use, and does anything look at what comes back before the agent reads it?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Agents connect to whichever tool servers their configuration names; there is no allow-list, no rate limit and nothing inspects what a tool sends back.", "3": "A gateway holds the deny-list and allow-list; unlisted tools and unauthenticated servers are refused; tool responses are scanned and blocked, sanitised or logged.", "5": "A scanner checks tools against a baseline for schema drift, typosquatting and poisoning and revokes trust on its own; the gate’s list is the register of skills and tools." }, "help": "Evidence that would show it: Gateway configuration: deny-list, allow-list, sensitive-tool approvals, rate limits; Response-scan log showing BLOCK, SANITIZE or LOG outcomes; Scanner report for tool poisoning and schema drift against a baseline.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-6", "type": "scored_text", "category": "identity", "name": "Kill switch and action limits", "text": "If one of your agents started doing something wrong at two in the morning, how would you stop it, how long would it take, and when did you last try? What is the most it could spend or send before anything noticed?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Stopping an agent means finding the engineer who deployed it; no budgets or rate limits exist and nobody has tried to switch one off in anger.", "3": "A named kill switch exists per agent with a dated test record showing it worked; rate, token and cost ceilings are enforced at runtime, with a named alert recipient.", "5": "Anomalous behaviour trips a circuit breaker or the kill switch without a person; budgets adjust from observed use; the kill switch is drilled on a schedule." }, "help": "Evidence that would show it: Kill-switch test record per agent, dated; Budget and rate-limit configuration per agent with alert recipient; Capability scope per agent, showing what it cannot do.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-7", "type": "scored_text", "category": "evidence", "name": "One enforcement point for model and tool calls", "text": "Draw me the path a request takes from your agent to the model, and from your agent to a tool. Are they the same path? Is there any way a tool gets called that skips the front door?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Model traffic goes through the API gateway but agents call tool servers directly, so those calls never meet a policy; a sidecar may exist that nothing calls.", "3": "Tool servers sit behind the gateway URL; a decision service answers allow, deny, conditions or require-approval per call; the gateway blocks on timeout or a bad response.", "5": "One decision service governs every hosting surface, including in-process middleware in Foundry, with one telemetry backend; no tool goes live except behind the gateway." }, "help": "Evidence that would show it: Gateway policy showing the decision-point call and fail-closed handling; List of tool servers registered behind the gateway URL; Telemetry from gateway and middleware landing in one monitoring workspace.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-8", "type": "scored_text", "category": "evidence", "name": "Governed developer assistants", "text": "What are your developers’ coding assistants allowed to run on their machines, and what stops one from running a script it just downloaded? Could you show me what one of them did yesterday?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Staff run coding assistants with whatever permissions the tool defaults to; nothing records what those assistants executed, and nobody knows which machines run them.", "3": "A policy is loaded on each staff machine; unknown tools are reviewed, downloaded scripts and credential access are blocked, and the local audit log reaches the register.", "5": "Profiles graduate from advisory to blocking on evidence per the two-week model; audit logs centralise at the gate; a machine without a policy shows up in the register." }, "help": "Evidence that would show it: Policy file and profile per assistant (strict, balanced or advisory); Local audit log samples, and the record of their ingestion into the register; Graduation record: advisory start date, clean weeks, blocking switch-on.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false }, { "id": "agt-9", "type": "scored_text", "category": "evidence", "name": "Tamper-evident audit and the Decision BOM", "text": "Pick any action an agent took last week. Could you show me why it was allowed, which rule said so, and who it was acting for? If someone altered that log, how would you know?", "scale": "maturity5", "scored": true, "star": false, "rubric": { "1": "Logs are whatever the framework prints; they can be edited, they do not name the agent or the rule, and nobody can say why a given action was permitted last Tuesday.", "3": "An append-only chained log exists; for any action a record shows the agent, the rules evaluated and the outcome, tied to the person who asked, and a tamper check passes.", "5": "Decision records are rebuilt in batch over a time window with a completeness score and feed the quarterly review; a completeness gap raises a finding on its own." }, "help": "Evidence that would show it: Audit log with hash-chain verification output; Sample Decision BOM for one action, with its completeness score; Quarterly report of verdict counts and deny rates drawn from the log.", "adaptive": { "allow_probe": true, "allow_skip": false, "max_probes": 1 }, "ai_drafted": false } ] } ], "grids": {}, "outputs": [ "Level per element and per category, gated", "Contested-element view (spread of 2 or more)", "Coverage of evidence: confirmed, stated, inferred", "Where to start, foundations first", "Printable client report" ], "report_defaults": [ "rpt-maturity-standard" ], "playbook": { "sequence": [ "policy", "identity", "evidence" ], "sequence_note": "From the intel file’s dated path: decide the enforcement pattern first, pilot the assistants in advisory mode, then deepen tier by tier. Do not adopt trust scoring, the mesh or rings yet.", "actions": { "policy": { "to_3": [ "Name the decision runtime in the platform design and write deny-list, allow-list, sensitive-tool approval and rate limit into it", "Add to the controls library: no host runs with an empty policy set, and a health check runs green on every governed host", "Configure an approval backend before enabling escalate, and render the approver a view the agent did not write" ], "to_5": [ "Feed verdict counts and deny rates into the quarterly health check and review the rules from them" ] }, "identity": { "to_3": [ "Give every agent its own directory identity, a named human sponsor and a register row; test that suspension stops it", "Register every skill and tool at the gate so the register is met by construction", "Test the kill switch per agent and set rate, token and cost budgets with a named alert recipient" ], "to_5": [ "Turn on shadow-agent discovery and access reviews so unregistered or sponsorless agents surface without a person looking" ] }, "evidence": { "to_3": [ "Register tool servers behind the gateway URL so tool calls meet the same enforcement point as model calls", "Pilot the two assistant plugins on a named team in advisory mode; graduate to blocking after two clean weeks", "Keep manifests in version control as documented information and verdict logs as operating evidence" ], "to_5": [ "Reconstruct Decision BOMs in batch for the assurance review and send all telemetry to one monitoring workspace" ] } } } }