Enum TaskSuite 

pub enum TaskSuite {
    Default,
    Custom {
        tasks: Vec<TuneTask>,
    },
}

The set of tasks a tune run evaluates each candidate against.

Custom carries the exact same JSON shape whether it originates from a CLI --task-suite path.json file or a GUI file upload parsed client-side and posted as part of the run request — there is a single shared schema, not two divergent ingestion paths.

Variants

Default

The built-in default suite (see assets/tune_default_suite.json).

Custom

A user-authored suite.

Fields

tasks: Vec<TuneTask>

Implementations

impl TaskSuite

const DEFAULT_SUITE_JSON: &'static str = "[\n {\n \"id\": \"single_call_weather\",\n \"category\": \"single_call\",\n \"user_prompt\": \"What\'s the current weather in Boston, MA?\",\n \"tools\": [\n {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a location.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\",\n \"description\": \"City and state, e.g. \'Boston, MA\'\"\n }\n },\n \"required\": [\"location\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"get_weather\",\n \"required_args\": { \"location\": \"Boston, MA\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_read_file\",\n \"category\": \"single_call\",\n \"user_prompt\": \"What does the file named config.json contain?\",\n \"tools\": [\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"read_file\",\n \"required_args\": { \"path\": \"config.json\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"parallel_call_two_cities\",\n \"category\": \"parallel_call\",\n \"user_prompt\": \"What\'s the weather like in both Boston and Austin right now?\",\n \"tools\": [\n {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a location.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": { \"type\": \"string\" }\n },\n \"required\": [\"location\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"get_weather\",\n \"required_args\": { \"location\": \"Boston\" },\n \"ordered\": false\n },\n {\n \"name\": \"get_weather\",\n \"required_args\": { \"location\": \"Austin\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"multi_turn_search_then_read\",\n \"category\": \"multi_turn\",\n \"user_prompt\": \"Find the file that defines the `AgentLoop` struct, then show me its contents.\",\n \"tools\": [\n {\n \"name\": \"search_files\",\n \"description\": \"Search the workspace for files matching a query.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": { \"type\": \"string\" }\n },\n \"required\": [\"query\"]\n }\n },\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"search_files\",\n \"required_args\": {},\n \"ordered\": true\n },\n {\n \"name\": \"read_file\",\n \"required_args\": {},\n \"ordered\": true,\n \"depends_on_result\": true\n }\n ]\n }\n },\n {\n \"id\": \"irrelevance_general_knowledge\",\n \"category\": \"irrelevance\",\n \"user_prompt\": \"What is the capital of France?\",\n \"tools\": [\n {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a location.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": { \"type\": \"string\" }\n },\n \"required\": [\"location\"]\n }\n },\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": { \"kind\": \"no_tool_call\" }\n },\n {\n \"id\": \"irrelevance_no_matching_tool\",\n \"category\": \"irrelevance\",\n \"user_prompt\": \"Please send an email to my manager summarizing today\'s standup.\",\n \"tools\": [\n {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a location.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": { \"type\": \"string\" }\n },\n \"required\": [\"location\"]\n }\n }\n ],\n \"expected\": { \"kind\": \"no_tool_call\" }\n },\n {\n \"id\": \"single_call_ambiguous_but_clear\",\n \"category\": \"single_call\",\n \"system_prompt\": \"You are a helpful coding assistant with access to filesystem tools.\",\n \"user_prompt\": \"Check whether the file named README.md exists in the current directory.\",\n \"tools\": [\n {\n \"name\": \"file_exists\",\n \"description\": \"Check whether a file exists at the given path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"file_exists\",\n \"required_args\": { \"path\": \"README.md\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"parallel_call_multi_file_read\",\n \"category\": \"parallel_call\",\n \"user_prompt\": \"Show me the contents of both package.json and Cargo.toml.\",\n \"tools\": [\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"read_file\",\n \"required_args\": { \"path\": \"package.json\" },\n \"ordered\": false\n },\n {\n \"name\": \"read_file\",\n \"required_args\": { \"path\": \"Cargo.toml\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"long_context_refactor_endurance\",\n \"category\": \"long_context\",\n \"system_prompt\": \"You are a helpful coding assistant with access to filesystem tools. Work efficiently and avoid repeating actions you have already completed.\",\n \"history\": [\n {\n \"role\": \"user\",\n \"content\": \"I\'m refactoring the `UserService` class to split it into `UserRepository` and `UserValidator`. Here\'s the current file:\\n\\n```python\\nclass UserService:\\n def __init__(self, db_connection):\\n self.db = db_connection\\n\\n def validate_email(self, email):\\n if \\\"@\\\" not in email:\\n raise ValueError(\\\"invalid email\\\")\\n local, _, domain = email.partition(\\\"@\\\")\\n if not local or \\\".\\\" not in domain:\\n raise ValueError(\\\"invalid email domain\\\")\\n return True\\n\\n def validate_password(self, password):\\n if len(password) < 8:\\n raise ValueError(\\\"password too short\\\")\\n if not any(c.isdigit() for c in password):\\n raise ValueError(\\\"password needs a digit\\\")\\n return True\\n\\n def get_user_by_id(self, user_id):\\n return self.db.execute(\\\"SELECT * FROM users WHERE id = ?\\\", (user_id,)).fetchone()\\n\\n def get_user_by_email(self, email):\\n return self.db.execute(\\\"SELECT * FROM users WHERE email = ?\\\", (email,)).fetchone()\\n\\n def create_user(self, email, password):\\n self.validate_email(email)\\n self.validate_password(password)\\n self.db.execute(\\\"INSERT INTO users (email, password) VALUES (?, ?)\\\", (email, password))\\n self.db.commit()\\n```\\n\\nLet\'s start by extracting the validation logic.\"\n },\n {\n \"role\": \"assistant\",\n \"tool_calls\": [\n { \"id\": \"call_1\", \"name\": \"read_file\", \"arguments\": { \"path\": \"user_service.py\" } }\n ]\n },\n {\n \"role\": \"tool\",\n \"tool_call_id\": \"call_1\",\n \"content\": \"class UserService:\\n def __init__(self, db_connection):\\n self.db = db_connection\\n # ... (full contents already shown above)\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I\'ve reviewed the file. I\'ll extract `validate_email` and `validate_password` into a new `UserValidator` class.\"\n },\n {\n \"role\": \"assistant\",\n \"tool_calls\": [\n {\n \"id\": \"call_2\",\n \"name\": \"edit_file\",\n \"arguments\": {\n \"path\": \"user_validator.py\",\n \"content\": \"class UserValidator:\\n @staticmethod\\n def validate_email(email):\\n if \\\"@\\\" not in email:\\n raise ValueError(\\\"invalid email\\\")\\n local, _, domain = email.partition(\\\"@\\\")\\n if not local or \\\".\\\" not in domain:\\n raise ValueError(\\\"invalid email domain\\\")\\n return True\\n\\n @staticmethod\\n def validate_password(password):\\n if len(password) < 8:\\n raise ValueError(\\\"password too short\\\")\\n if not any(c.isdigit() for c in password):\\n raise ValueError(\\\"password needs a digit\\\")\\n return True\\n\"\n }\n }\n ]\n },\n {\n \"role\": \"tool\",\n \"tool_call_id\": \"call_2\",\n \"content\": \"File user_validator.py created successfully.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Great, now let\'s also extract the database access logic into `UserRepository`.\"\n },\n {\n \"role\": \"assistant\",\n \"tool_calls\": [\n {\n \"id\": \"call_3\",\n \"name\": \"edit_file\",\n \"arguments\": {\n \"path\": \"user_repository.py\",\n \"content\": \"class UserRepository:\\n def __init__(self, db_connection):\\n self.db = db_connection\\n\\n def get_user_by_id(self, user_id):\\n return self.db.execute(\\\"SELECT * FROM users WHERE id = ?\\\", (user_id,)).fetchone()\\n\\n def get_user_by_email(self, email):\\n return self.db.execute(\\\"SELECT * FROM users WHERE email = ?\\\", (email,)).fetchone()\\n\\n def insert_user(self, email, password):\\n self.db.execute(\\\"INSERT INTO users (email, password) VALUES (?, ?)\\\", (email, password))\\n self.db.commit()\\n\"\n }\n }\n ]\n },\n {\n \"role\": \"tool\",\n \"tool_call_id\": \"call_3\",\n \"content\": \"File user_repository.py created successfully.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Now simplify `UserService` to delegate to both new classes instead of duplicating their logic.\"\n },\n {\n \"role\": \"assistant\",\n \"tool_calls\": [\n {\n \"id\": \"call_4\",\n \"name\": \"edit_file\",\n \"arguments\": {\n \"path\": \"user_service.py\",\n \"content\": \"class UserService:\\n def __init__(self, db_connection):\\n self.repository = UserRepository(db_connection)\\n self.validator = UserValidator()\\n\\n def create_user(self, email, password):\\n self.validator.validate_email(email)\\n self.validator.validate_password(password)\\n self.repository.insert_user(email, password)\\n\"\n }\n }\n ]\n },\n {\n \"role\": \"tool\",\n \"tool_call_id\": \"call_4\",\n \"content\": \"File user_service.py updated successfully.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Done \u{2014} `UserRepository` now owns all database access, `UserValidator` owns validation, and `UserService` delegates to both. Let me know what\'s next.\"\n }\n ],\n \"user_prompt\": \"Now check whether there\'s an existing test file for `UserService` so we can update it to match the new structure.\",\n \"tools\": [\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n },\n {\n \"name\": \"edit_file\",\n \"description\": \"Create or overwrite a file with the given content.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" },\n \"content\": { \"type\": \"string\" }\n },\n \"required\": [\"path\", \"content\"]\n }\n },\n {\n \"name\": \"file_exists\",\n \"description\": \"Check whether a file exists at the given path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": { \"type\": \"string\" }\n },\n \"required\": [\"path\"]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"file_exists\",\n \"required_args\": { \"path\": \"test_user_service.py\" },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_typed_args\",\n \"category\": \"single_call\",\n \"user_prompt\": \"Show me the last forty-two lines of the file at logs/proxy.log, and make the search case sensitive.\",\n \"tools\": [\n {\n \"name\": \"read_log_lines\",\n \"description\": \"Read the last N lines of a log file, optionally case-sensitively.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\"\n },\n \"case_sensitive\": {\n \"type\": \"boolean\"\n }\n },\n \"required\": [\n \"path\",\n \"count\",\n \"case_sensitive\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"read_log_lines\",\n \"required_args\": {\n \"path\": \"logs/proxy.log\",\n \"count\": 42,\n \"case_sensitive\": true\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_nested_options\",\n \"category\": \"single_call\",\n \"user_prompt\": \"List the src directory recursively, but only two levels deep.\",\n \"tools\": [\n {\n \"name\": \"list_directory\",\n \"description\": \"List a directory. The options object controls traversal: recursive (boolean) and max_depth (integer).\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"options\": {\n \"type\": \"object\",\n \"properties\": {\n \"recursive\": {\n \"type\": \"boolean\"\n },\n \"max_depth\": {\n \"type\": \"integer\"\n }\n },\n \"required\": [\n \"recursive\",\n \"max_depth\"\n ]\n }\n },\n \"required\": [\n \"path\",\n \"options\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"list_directory\",\n \"required_args\": {\n \"path\": \"src\",\n \"options\": {\n \"recursive\": true,\n \"max_depth\": 2\n }\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_verbatim_payload\",\n \"category\": \"single_call\",\n \"user_prompt\": \"Create a file named notes/snippet.txt containing exactly this line, byte for byte:\\ntemplate = \\\"</parameter>\\\" marks the end of {arg} blocks\\nDo not rewrite, escape, or annotate it.\",\n \"tools\": [\n {\n \"name\": \"write_file\",\n \"description\": \"Write content to a file, replacing whatever is there.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"content\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\",\n \"content\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"write_file\",\n \"required_args\": {\n \"path\": \"notes/snippet.txt\",\n \"content\": \"template = \\\"</parameter>\\\" marks the end of {arg} blocks\"\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_pick_the_right_tool\",\n \"category\": \"single_call\",\n \"user_prompt\": \"Where does the user guide explain retry budgets? Search the documentation for it.\",\n \"tools\": [\n {\n \"name\": \"search_code\",\n \"description\": \"Search source code files for a query.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"query\"\n ]\n }\n },\n {\n \"name\": \"search_docs\",\n \"description\": \"Search the documentation for a query.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"query\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"search_docs\",\n \"required_args\": {},\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"single_call_derived_value\",\n \"category\": \"single_call\",\n \"user_prompt\": \"Set a timer for two and a half hours.\",\n \"tools\": [\n {\n \"name\": \"set_timer\",\n \"description\": \"Set a countdown timer. Duration is in minutes.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"minutes\": {\n \"type\": \"integer\"\n }\n },\n \"required\": [\n \"minutes\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"set_timer\",\n \"required_args\": {\n \"minutes\": 150\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"parallel_call_paired_forecasts\",\n \"category\": \"parallel_call\",\n \"user_prompt\": \"Get the forecast for Oslo on Friday, Lisbon on Saturday, and Kyoto on Sunday.\",\n \"tools\": [\n {\n \"name\": \"get_forecast\",\n \"description\": \"Get the weather forecast for a city on a given day.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n },\n \"day\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\",\n \"day\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"get_forecast\",\n \"required_args\": {\n \"city\": \"Oslo\",\n \"day\": \"Friday\"\n },\n \"ordered\": false\n },\n {\n \"name\": \"get_forecast\",\n \"required_args\": {\n \"city\": \"Lisbon\",\n \"day\": \"Saturday\"\n },\n \"ordered\": false\n },\n {\n \"name\": \"get_forecast\",\n \"required_args\": {\n \"city\": \"Kyoto\",\n \"day\": \"Sunday\"\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"parallel_call_mixed_tools\",\n \"category\": \"parallel_call\",\n \"user_prompt\": \"I\'m heading out: check the current weather in Oslo and read me the file TODO.md.\",\n \"tools\": [\n {\n \"name\": \"get_weather\",\n \"description\": \"Get the current weather for a location.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"location\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"location\"\n ]\n }\n },\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"get_weather\",\n \"required_args\": {\n \"location\": \"Oslo\"\n },\n \"ordered\": false\n },\n {\n \"name\": \"read_file\",\n \"required_args\": {\n \"path\": \"TODO.md\"\n },\n \"ordered\": false\n }\n ]\n }\n },\n {\n \"id\": \"multi_turn_create_then_append\",\n \"category\": \"multi_turn\",\n \"user_prompt\": \"Create an empty file at notes/todo.md, and once that succeeds, append the line \'ship the eval gates\' to it.\",\n \"tools\": [\n {\n \"name\": \"create_file\",\n \"description\": \"Create an empty file at a path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\"\n ]\n }\n },\n {\n \"name\": \"append_file\",\n \"description\": \"Append a line of text to an existing file.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n },\n \"line\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\",\n \"line\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"create_file\",\n \"required_args\": {\n \"path\": \"notes/todo.md\"\n },\n \"ordered\": true\n },\n {\n \"name\": \"append_file\",\n \"required_args\": {\n \"path\": \"notes/todo.md\",\n \"line\": \"ship the eval gates\"\n },\n \"ordered\": true\n }\n ]\n }\n },\n {\n \"id\": \"multi_turn_check_then_delete\",\n \"category\": \"multi_turn\",\n \"user_prompt\": \"Check whether build/cache.tmp exists, and if the check comes back fine, delete that file.\",\n \"tools\": [\n {\n \"name\": \"file_exists\",\n \"description\": \"Check whether a file exists at a path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\"\n ]\n }\n },\n {\n \"name\": \"delete_file\",\n \"description\": \"Delete a file at a path. Irreversible.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"file_exists\",\n \"required_args\": {\n \"path\": \"build/cache.tmp\"\n },\n \"ordered\": true\n },\n {\n \"name\": \"delete_file\",\n \"required_args\": {\n \"path\": \"build/cache.tmp\"\n },\n \"ordered\": true,\n \"depends_on_result\": true\n }\n ]\n }\n },\n {\n \"id\": \"irrelevance_tempting_filename\",\n \"category\": \"irrelevance\",\n \"user_prompt\": \"What does the \'.rs\' in a file name like main.rs stand for, and which language uses it?\",\n \"tools\": [\n {\n \"name\": \"read_file\",\n \"description\": \"Read the full contents of a file by path.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"path\"\n ]\n }\n },\n {\n \"name\": \"search_files\",\n \"description\": \"Search the workspace for files matching a query.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"query\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"query\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"no_tool_call\"\n }\n },\n {\n \"id\": \"irrelevance_tool_scope_mismatch\",\n \"category\": \"irrelevance\",\n \"user_prompt\": \"What was Apple\'s stock price in March 1990?\",\n \"tools\": [\n {\n \"name\": \"get_stock_price\",\n \"description\": \"Get the current, real-time stock price for a ticker symbol. No historical data.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"symbol\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"symbol\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"no_tool_call\"\n }\n },\n {\n \"id\": \"long_context_planted_values\",\n \"category\": \"long_context\",\n \"history\": [\n {\n \"role\": \"user\",\n \"content\": \"Let\'s plan the staging deploy for the metrics collector. I want to get every detail agreed before we touch anything.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Understood \u{2014} tell me the constraints and I will keep track of what we decide.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"First decision: the service listens on port 8443. Not 8080 like the old collector \u{2014} security wants TLS termination on the service itself this time.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Noted: port 8443, TLS terminated at the service rather than at the ingress. What host will it run on?\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Before that \u{2014} a digression. The old collector kept falling over because its queue was unbounded. The new one caps the queue at 10k entries and sheds load above that. That is config, not deploy, so it is not your concern today.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Understood, queue sizing is out of scope for the deploy. Back to the host, then.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"The host is metrics-02.staging.internal. metrics-01 is still running the old collector until the cutover, so do not touch it.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Noted: metrics-02.staging.internal, and metrics-01 stays untouched. Anything else before we register it?\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Two more things. The health check path is /healthz \u{2014} the default /health collides with an existing route. And name the service metrics-collector-v2 in the registry; plain metrics-collector is the old one.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"All recorded: name metrics-collector-v2, host metrics-02.staging.internal, port 8443, health check at /healthz. Say the word when you want it registered.\"\n },\n {\n \"role\": \"user\",\n \"content\": \"One last digression: remind me Monday to write the runbook for the cutover. Not now \u{2014} just do not let me forget it.\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I will bring the runbook up when we next talk. Ready to register when you are.\"\n }\n ],\n \"user_prompt\": \"Go ahead \u{2014} register the service with everything we agreed.\",\n \"tools\": [\n {\n \"name\": \"register_service\",\n \"description\": \"Register a service in the staging registry.\",\n \"input_schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"name\": {\n \"type\": \"string\"\n },\n \"host\": {\n \"type\": \"string\"\n },\n \"port\": {\n \"type\": \"integer\"\n },\n \"health_path\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"name\",\n \"host\",\n \"port\",\n \"health_path\"\n ]\n }\n }\n ],\n \"expected\": {\n \"kind\": \"tool_calls\",\n \"calls\": [\n {\n \"name\": \"register_service\",\n \"required_args\": {\n \"name\": \"metrics-collector-v2\",\n \"host\": \"metrics-02.staging.internal\",\n \"port\": 8443,\n \"health_path\": \"/healthz\"\n },\n \"ordered\": false\n }\n ]\n }\n }\n]\n"

Embedded JSON for the built-in default suite (BFCL-style: single-call, parallel-call, multi-turn, and irrelevance-detection scenarios, plus a long-context endurance scenario).

pub fn resolve(&self) -> Result<Vec<TuneTask>, Error>

Resolve this suite into its concrete list of tasks.

Errors

Returns an error only for TaskSuite::Default, and only if the embedded JSON asset is malformed — that would indicate a build-time bug in gglib itself, never a user input error. TaskSuite::Custom never errors here (its tasks were already deserialized when the TaskSuite value itself was parsed).

Trait Implementations

impl Clone for TaskSuite

fn clone(&self) -> TaskSuite

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for TaskSuite

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for TaskSuite

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for TaskSuite

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.