{ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { "type": "string", "description": "The name of the playbook." }, "agent_model": { "type": "string", "description": "The model the Gemini CLI should use (e.g., gemini-2.5-pro)." }, "evaluator_model": { "type": "string", "description": "The model the test harness uses to grade the test and simulate the user (e.g., gemini-2.5-flash)." }, "tmpdir": { "type": "object", "description": "Configuration for running in a temporary isolated workspace with optional symlinks.", "properties": { "link_paths": { "type": "array", "description": "Relative paths to symlink from the host repository (current CWD) into the temporary workspace.", "items": { "type": "string" } } }, "additionalProperties": false }, "timeout": { "type": "integer", "description": "Timeout in seconds for each CLI invocation.", "default": 60, "minimum": 1 }, "env": { "type": "array", "description": "A list of environment variable names required by this playbook. These will be substituted in the steps and persona context.", "items": { "type": "string" } }, "steps": { "type": "array", "description": "The deterministic sequence of interactions between the user and the agent.", "items": { "type": "object", "properties": { "user_input": { "type": "string", "description": "The simulated input provided by the user." }, "expected_outcome": { "type": "string", "description": "The expected response or behavior from the agent to evaluate against." } }, "required": [ "user_input", "expected_outcome" ], "additionalProperties": false } }, "persona": { "type": "object", "description": "Configuration for the autonomous LLM-simulated user.", "properties": { "initial_user_input": { "type": "string", "description": "The first input to send to the agent when starting in pure autonomous mode. Variables are interpolated." }, "context": { "type": "string", "description": "Freeform instructions and knowledge base for the simulated user. Variables are interpolated." }, "max_turns": { "type": "integer", "description": "The maximum number of conversation turns allowed in autonomous mode before forcing a failure.", "default": 10, "minimum": 1 }, "success_criteria": { "type": "object", "description": "The conditions that must be met for the autonomous flow to be considered complete and successful.", "properties": { "llm_checks": { "type": "array", "description": "Semantic checks evaluated by the LLM (e.g., 'The agent printed a final configuration summary').", "items": { "type": "string" } }, "flow_contains": { "type": "array", "description": "Literal strings that must appear somewhere in the combined CLI stdout.", "items": { "type": "string" } }, "files_exist": { "type": "array", "description": "A list of file paths (relative to the workspace) that must exist.", "items": { "type": "string" } }, "files_contain": { "type": "object", "description": "A mapping of file paths to a list of strings that must be found within them.", "patternProperties": { ".*": { "type": "array", "items": { "type": "string" } } } }, "tool_calls_contain": { "type": "object", "description": "A mapping of tool names to a list of strings that must be found within their arguments.", "patternProperties": { ".*": { "type": "array", "items": { "type": "string" } } } } }, "additionalProperties": false } }, "required": [ "context", "success_criteria" ], "additionalProperties": false } }, "required": [ "name" ], "anyOf": [ { "required": ["steps"] }, { "required": ["persona"] } ], "if": { "not": { "required": ["steps"] } }, "then": { "properties": { "persona": { "required": ["context", "success_criteria", "initial_user_input"] } } }, "additionalProperties": false }