gglib_core/domain/benchmark/tune/config.rs
1//! Tune-run configuration: sweep specification and scoring weights.
2
3use serde::{Deserialize, Serialize};
4
5use super::task::TaskSuite;
6
7/// Configuration for a tune benchmark run.
8///
9/// A tune run evaluates one model against many candidate `InferenceConfig`
10/// sampling settings, scoring each candidate against an agentic tool-calling
11/// task suite to find the settings that make the model both accurate at
12/// tool calls and resistant to loop/stagnation guard triggers.
13#[derive(Debug, Clone, Serialize, Deserialize)]
14pub struct TuneConfig {
15 /// Database ID of the model to tune.
16 pub model_id: i64,
17 /// Task suite to evaluate candidates against.
18 pub task_suite: TaskSuite,
19 /// Sampling-parameter values to sweep (cartesian product forms the grid).
20 pub sweep: SweepSpec,
21 /// Seed additional candidates from the built-in per-model-family preset
22 /// table (e.g. Qwen coding-mode defaults).
23 #[serde(default = "SweepSpec::default_true")]
24 pub seed_from_family_presets: bool,
25 /// Weights used to combine per-candidate metrics into a composite score.
26 ///
27 /// `None` means "the server decides", and is what a client sends when the
28 /// user named no weights. It is not the same as sending the defaults: a
29 /// client that spells them out pins its own copy, and an older server
30 /// receiving a weights object it cannot parse rejects the whole request.
31 ///
32 /// `skip_serializing_if` is load-bearing, not tidiness. `None` would
33 /// otherwise go out as `"weights": null`, and to a server where this
34 /// field is still a plain `ScoreWeights` that is a type error rather than
35 /// an absent key — the one spelling that is compatible in both directions
36 /// is no key at all.
37 #[serde(default, skip_serializing_if = "Option::is_none")]
38 pub weights: Option<ScoreWeights>,
39 /// Fraction of candidates dropped after the cheap pre-screen round.
40 ///
41 /// `0.5` drops the bottom half of candidates before running the full
42 /// task suite on the remaining survivors. Clamped to `[0.0, 0.9]`.
43 #[serde(default = "TuneConfig::default_prune_fraction")]
44 pub prune_fraction: f32,
45 /// Override the llama-server context window size for this run.
46 #[serde(default)]
47 pub ctx_size: Option<u64>,
48}
49
50impl TuneConfig {
51 const fn default_prune_fraction() -> f32 {
52 0.5
53 }
54}
55
56/// Sampling-parameter values to sweep.
57///
58/// Each field is a list of candidate values for that dimension. The full
59/// candidate grid is the cartesian product of all non-empty dimensions; an
60/// empty list means "don't vary this dimension" (the resolved default from
61/// the normal inference-config fallback chain is used instead).
62#[derive(Debug, Clone, Default, Serialize, Deserialize)]
63pub struct SweepSpec {
64 /// Candidate temperature values.
65 #[serde(default)]
66 pub temperature: Vec<f32>,
67 /// Candidate top-p (nucleus sampling) values.
68 #[serde(default)]
69 pub top_p: Vec<f32>,
70 /// Candidate top-k values.
71 #[serde(default)]
72 pub top_k: Vec<i32>,
73 /// Candidate min-p values.
74 #[serde(default)]
75 pub min_p: Vec<f32>,
76 /// Candidate repeat-penalty values.
77 #[serde(default)]
78 pub repeat_penalty: Vec<f32>,
79 /// Candidate DRY multiplier values. `0.0` disables DRY, so a sweep of
80 /// `0.0,0.4,0.8` measures "off" against two strengths in one run.
81 ///
82 /// Only the multiplier is a dimension. `dry_base`, `dry_allowed_length`
83 /// and `dry_penalty_last_n` keep llama.cpp's defaults (1.75, 2, 64):
84 /// varying all four would multiply the grid by 81 for parameters whose
85 /// shipped values are already reasonable.
86 #[serde(default)]
87 pub dry_multiplier: Vec<f32>,
88 /// Candidate dynatemp half-range values. `0.0` disables dynamic
89 /// temperature, so `0.0,0.4` measures "off" against one strength — the
90 /// direct comparison of a flat temperature against an entropy-adaptive
91 /// band around the same base.
92 #[serde(default)]
93 pub dynatemp_range: Vec<f32>,
94 /// Candidate dynatemp exponent values. Only meaningful in a grid that
95 /// also sweeps (or fixes) a non-zero `dynatemp_range`; llama.cpp's
96 /// default is 1.0.
97 #[serde(default)]
98 pub dynatemp_exponent: Vec<f32>,
99 /// Candidate top-n-sigma values. `-1.0` disables the truncation, so
100 /// `-1.0,1.0` measures "off" against the paper's lower bound in one run.
101 #[serde(default)]
102 pub top_n_sigma: Vec<f32>,
103}
104
105impl SweepSpec {
106 const fn default_true() -> bool {
107 true
108 }
109}
110
111/// Weights used to combine per-candidate metrics into one composite score.
112///
113/// Each weight should be non-negative; the service normalizes the weighted
114/// sum by the total weight, so the three values do not need to sum to `1.0`.
115#[derive(Debug, Clone, Serialize, Deserialize)]
116pub struct ScoreWeights {
117 /// Weight applied to the average tool-call match score (AST-style,
118 /// partial credit) across all tasks in the suite.
119 pub tool_accuracy: f32,
120 /// Weight applied to `1 - (loop/stagnation trigger rate)`.
121 pub loop_avoidance: f32,
122 /// Weight applied to the fraction of tasks the agent completed
123 /// (produced a final answer instead of erroring out).
124 pub task_completion: f32,
125}
126
127impl ScoreWeights {
128 /// Weighted mean of the three axes, renormalized over those that were
129 /// measured. An unmeasured loop-avoidance axis claims no weight rather than
130 /// scoring `0.0` or an imputed `1.0`.
131 ///
132 /// The single definition of the composite, because the number is computed
133 /// in two places that must not drift: once per arm from its own runs, and
134 /// again when two arms are compared over the axes they share. A second copy
135 /// of this arithmetic is how a comparison starts measuring the scale
136 /// instead of the pipeline.
137 #[must_use]
138 pub fn composite_of(
139 &self,
140 tool_accuracy: f64,
141 loop_avoidance: Option<f64>,
142 task_completion: f64,
143 ) -> f64 {
144 let (loop_term, loop_weight) = loop_avoidance.map_or((0.0, 0.0), |avoidance| {
145 (
146 avoidance * f64::from(self.loop_avoidance),
147 f64::from(self.loop_avoidance),
148 )
149 });
150
151 let weight_sum =
152 f64::from(self.tool_accuracy) + loop_weight + f64::from(self.task_completion);
153 if weight_sum <= 0.0 {
154 return 0.0;
155 }
156
157 // Nested `mul_add` rather than the plain sum of products: same value,
158 // and it is what the workspace's float lints ask for.
159 let weighted = task_completion.mul_add(
160 f64::from(self.task_completion),
161 tool_accuracy.mul_add(f64::from(self.tool_accuracy), loop_term),
162 );
163 weighted / weight_sum
164 }
165}
166
167impl Default for ScoreWeights {
168 /// Prioritizes tool-call correctness over loop-avoidance and completion,
169 /// reflecting that an agentic backend which mis-calls tools is unusable
170 /// regardless of how it scores elsewhere.
171 fn default() -> Self {
172 Self {
173 tool_accuracy: 0.4,
174 loop_avoidance: 0.3,
175 task_completion: 0.2,
176 }
177 }
178}
179
180#[cfg(test)]
181mod tests {
182 use super::*;
183
184 /// `None` must leave the key out of the request body, not write `null`
185 /// into it. The two are the same to this server — `#[serde(default)]`
186 /// fires for an absent key, and `Option` accepts an explicit `null` — but
187 /// they are not the same to an older one, where this field is a plain
188 /// `ScoreWeights` rather than an `Option`. There too an absent key is
189 /// legal and gets that server's own default; a present `null` is a type
190 /// error that rejects the whole request.
191 ///
192 /// `Option<T>` serializes as `null` by default, so nothing but
193 /// `skip_serializing_if` produces the absent spelling, and no type check
194 /// notices the difference. The GUI's equivalent guarantee is asserted in
195 /// `tests/ts/components/TuneConfigForm.test.tsx`.
196 #[test]
197 fn a_config_with_no_weights_omits_the_key_rather_than_nulling_it() {
198 let config: TuneConfig = serde_json::from_str(
199 r#"{"model_id": 1, "task_suite": {"source": "default"}, "sweep": {}}"#,
200 )
201 .expect("minimal body deserializes");
202 assert!(config.weights.is_none());
203
204 let body = serde_json::to_value(&config).expect("serializes");
205 assert!(
206 !body.as_object().expect("object").contains_key("weights"),
207 "serialized body must not carry a `weights` key: {body}"
208 );
209 }
210}