hunfabric/tools/skill-turn-harness/playbooks/playbook.schema.json

{
  "$schema": "http://json-schema.org/draft-07/schema#",
  "type": "object",
  "properties": {
    "name": {
      "type": "string",
      "description": "The name of the playbook."
    },
    "agent_model": {
      "type": "string",
      "description": "The model the Gemini CLI should use (e.g., gemini-2.5-pro)."
    },
    "evaluator_model": {
      "type": "string",
      "description": "The model the test harness uses to grade the test and simulate the user (e.g., gemini-2.5-flash)."
    },
    "tmpdir": {
      "type": "object",
      "description": "Configuration for running in a temporary isolated workspace with optional symlinks.",
      "properties": {
        "link_paths": {
          "type": "array",
          "description": "Relative paths to symlink from the host repository (current CWD) into the temporary workspace.",
          "items": {
            "type": "string"
          }
        }
      },
      "additionalProperties": false
    },
    "timeout": {
      "type": "integer",
      "description": "Timeout in seconds for each CLI invocation.",
      "default": 60,
      "minimum": 1
    },
    "env": {
      "type": "array",
      "description": "A list of environment variable names required by this playbook. These will be substituted in the steps and persona context.",
      "items": {
        "type": "string"
      }
    },
    "steps": {
      "type": "array",
      "description": "The deterministic sequence of interactions between the user and the agent.",
      "items": {
        "type": "object",
        "properties": {
          "user_input": {
            "type": "string",
            "description": "The simulated input provided by the user."
          },
          "expected_outcome": {
            "type": "string",
            "description": "The expected response or behavior from the agent to evaluate against."
          }
        },
        "required": [
          "user_input",
          "expected_outcome"
        ],
        "additionalProperties": false
      }
    },
    "persona": {
      "type": "object",
      "description": "Configuration for the autonomous LLM-simulated user.",
      "properties": {
        "initial_user_input": {
          "type": "string",
          "description": "The first input to send to the agent when starting in pure autonomous mode. Variables are interpolated."
        },
        "context": {
          "type": "string",
          "description": "Freeform instructions and knowledge base for the simulated user. Variables are interpolated."
        },
        "max_turns": {
          "type": "integer",
          "description": "The maximum number of conversation turns allowed in autonomous mode before forcing a failure.",
          "default": 10,
          "minimum": 1
        },
        "success_criteria": {
          "type": "object",
          "description": "The conditions that must be met for the autonomous flow to be considered complete and successful.",
          "properties": {
            "llm_checks": {
              "type": "array",
              "description": "Semantic checks evaluated by the LLM (e.g., 'The agent printed a final configuration summary').",
              "items": {
                "type": "string"
              }
            },
            "flow_contains": {
              "type": "array",
              "description": "Literal strings that must appear somewhere in the combined CLI stdout.",
              "items": {
                "type": "string"
              }
            },
            "files_exist": {
              "type": "array",
              "description": "A list of file paths (relative to the workspace) that must exist.",
              "items": {
                "type": "string"
              }
            },
            "files_contain": {
              "type": "object",
              "description": "A mapping of file paths to a list of strings that must be found within them.",
              "patternProperties": {
                ".*": {
                  "type": "array",
                  "items": {
                    "type": "string"
                  }
                }
              }
            },
            "tool_calls_contain": {
              "type": "object",
              "description": "A mapping of tool names to a list of strings that must be found within their arguments.",
              "patternProperties": {
                ".*": {
                  "type": "array",
                  "items": {
                    "type": "string"
                  }
                }
              }
            }
          },
          "additionalProperties": false
        }
      },
      "required": [
        "context",
        "success_criteria"
      ],
      "additionalProperties": false
    }
  },
  "required": [
    "name"
  ],
  "anyOf": [
    {
      "required": ["steps"]
    },
    {
      "required": ["persona"]
    }
  ],
  "if": {
    "not": {
      "required": ["steps"]
    }
  },
  "then": {
    "properties": {
      "persona": {
        "required": ["context", "success_criteria", "initial_user_input"]
      }
    }
  },
  "additionalProperties": false
}