gglib_core/domain/benchmark/tune/
config.rs

1//! Tune-run configuration: sweep specification and scoring weights.
2
3use serde::{Deserialize, Serialize};
4
5use super::task::TaskSuite;
6
7/// Configuration for a tune benchmark run.
8///
9/// A tune run evaluates one model against many candidate `InferenceConfig`
10/// sampling settings, scoring each candidate against an agentic tool-calling
11/// task suite to find the settings that make the model both accurate at
12/// tool calls and resistant to loop/stagnation guard triggers.
13#[derive(Debug, Clone, Serialize, Deserialize)]
14pub struct TuneConfig {
15    /// Database ID of the model to tune.
16    pub model_id: i64,
17    /// Task suite to evaluate candidates against.
18    pub task_suite: TaskSuite,
19    /// Sampling-parameter values to sweep (cartesian product forms the grid).
20    pub sweep: SweepSpec,
21    /// Seed additional candidates from the model's GGUF metadata
22    /// author-recommended sampling defaults, when present.
23    #[serde(default = "SweepSpec::default_true")]
24    pub seed_from_gguf: bool,
25    /// Seed additional candidates from the built-in per-model-family preset
26    /// table (e.g. Qwen coding-mode defaults).
27    #[serde(default = "SweepSpec::default_true")]
28    pub seed_from_family_presets: bool,
29    /// Weights used to combine per-candidate metrics into a composite score.
30    #[serde(default)]
31    pub weights: ScoreWeights,
32    /// Fraction of candidates dropped after the cheap pre-screen round.
33    ///
34    /// `0.5` drops the bottom half of candidates before running the full
35    /// task suite on the remaining survivors. Clamped to `[0.0, 0.9]`.
36    #[serde(default = "TuneConfig::default_prune_fraction")]
37    pub prune_fraction: f32,
38    /// Override the llama-server context window size for this run.
39    #[serde(default)]
40    pub ctx_size: Option<u64>,
41}
42
43impl TuneConfig {
44    const fn default_prune_fraction() -> f32 {
45        0.5
46    }
47}
48
49/// Sampling-parameter values to sweep.
50///
51/// Each field is a list of candidate values for that dimension. The full
52/// candidate grid is the cartesian product of all non-empty dimensions; an
53/// empty list means "don't vary this dimension" (the resolved default from
54/// the normal inference-config fallback chain is used instead).
55#[derive(Debug, Clone, Default, Serialize, Deserialize)]
56pub struct SweepSpec {
57    /// Candidate temperature values.
58    #[serde(default)]
59    pub temperature: Vec<f32>,
60    /// Candidate top-p (nucleus sampling) values.
61    #[serde(default)]
62    pub top_p: Vec<f32>,
63    /// Candidate top-k values.
64    #[serde(default)]
65    pub top_k: Vec<i32>,
66    /// Candidate min-p values.
67    #[serde(default)]
68    pub min_p: Vec<f32>,
69    /// Candidate repeat-penalty values.
70    #[serde(default)]
71    pub repeat_penalty: Vec<f32>,
72}
73
74impl SweepSpec {
75    const fn default_true() -> bool {
76        true
77    }
78}
79
80/// Weights used to combine per-candidate metrics into one composite score.
81///
82/// Each weight should be non-negative; the service normalizes the weighted
83/// sum by the total weight, so the four values do not need to sum to `1.0`.
84#[derive(Debug, Clone, Serialize, Deserialize)]
85pub struct ScoreWeights {
86    /// Weight applied to the average tool-call match score (AST-style,
87    /// partial credit) across all tasks in the suite.
88    pub tool_accuracy: f32,
89    /// Weight applied to `1 - (loop/stagnation trigger rate)`.
90    pub loop_avoidance: f32,
91    /// Weight applied to the fraction of tasks the agent completed
92    /// (produced a final answer instead of erroring out).
93    pub task_completion: f32,
94    /// Weight applied to token-generation throughput, normalized against
95    /// the fastest candidate in the same run.
96    pub speed: f32,
97}
98
99impl Default for ScoreWeights {
100    /// Prioritizes tool-call correctness and loop-avoidance over raw speed,
101    /// reflecting that an agentic backend which loops or mis-calls tools is
102    /// unusable regardless of how fast it streams tokens.
103    fn default() -> Self {
104        Self {
105            tool_accuracy: 0.4,
106            loop_avoidance: 0.3,
107            task_completion: 0.2,
108            speed: 0.1,
109        }
110    }
111}
gglib_core/domain/benchmark/tune/config.rs

gglib_core/domain/benchmark/tune/
config.rs
