gglib_core/domain/benchmark/tune/
result.rs

1//! Per-candidate and per-task results produced by a tune run.
2
3use serde::{Deserialize, Serialize};
4
5use crate::domain::inference::InferenceConfig;
6
7use super::task::TaskCategory;
8
9/// Where a tune candidate's sampling settings came from.
10#[derive(Debug, Clone, Serialize, Deserialize)]
11#[serde(tag = "kind", rename_all = "snake_case")]
12pub enum CandidateSource {
13    /// One point in the user-specified [`super::config::SweepSpec`] grid.
14    UserGrid,
15    /// Seeded from the model's GGUF metadata author-recommended sampling
16    /// defaults, when present.
17    GgufAuthorDefault,
18    /// Seeded from the built-in per-model-family preset table (e.g. Qwen
19    /// coding-mode defaults).
20    FamilyPreset {
21        /// Display name of the matched family/preset (e.g. `"qwen-coding"`).
22        family: String,
23    },
24}
25
26#[cfg(test)]
27mod tests {
28    use super::*;
29
30    /// `CandidateSource` is `#[serde(tag = "kind")]` (internally tagged),
31    /// which only supports newtype variants whose inner value serializes as
32    /// a JSON object/map. `FamilyPreset` must therefore stay a *struct*
33    /// variant (`{ family: String }`), never a bare `FamilyPreset(String)`
34    /// newtype — the latter fails at serialization time with "cannot
35    /// serialize tagged newtype variant ... containing a string".
36    #[test]
37    fn candidate_source_family_preset_round_trips() {
38        let source = CandidateSource::FamilyPreset {
39            family: "qwen-coding".to_string(),
40        };
41        let json = serde_json::to_string(&source).expect("serializes");
42        let round_tripped: CandidateSource = serde_json::from_str(&json).expect("deserializes");
43        assert!(matches!(
44            round_tripped,
45            CandidateSource::FamilyPreset { .. }
46        ));
47    }
48
49    #[test]
50    fn candidate_source_unit_variants_round_trip() {
51        for source in [
52            CandidateSource::UserGrid,
53            CandidateSource::GgufAuthorDefault,
54        ] {
55            let json = serde_json::to_string(&source).expect("serializes");
56            let _: CandidateSource = serde_json::from_str(&json).expect("deserializes");
57        }
58    }
59}
60
61/// Result of evaluating one task against one candidate's sampling settings.
62#[derive(Debug, Clone, Serialize, Deserialize)]
63pub struct TuneTaskResult {
64    /// ID of the [`super::task::TuneTask`] this result corresponds to.
65    pub task_id: String,
66    /// Category the task belongs to (carried for leaderboard grouping).
67    pub category: TaskCategory,
68    /// `true` if the agent loop completed and its tool calls matched the
69    /// task's expected outcome (for `NoToolCall` tasks: no call was made).
70    pub passed: bool,
71    /// AST-style match score against the expected outcome, `0.0`–`1.0`.
72    ///
73    /// Partial credit: e.g. right tool name but a missing required
74    /// argument scores between `0.0` and `1.0`, not a hard fail.
75    pub tool_match_score: f64,
76    /// `true` if the agent loop's `LoopDetector` fired during this task.
77    pub loop_detected: bool,
78    /// `true` if the agent loop's `StagnationDetector` fired during this task.
79    pub stagnation_detected: bool,
80    /// Number of agent-loop iterations consumed before completion (or
81    /// before the loop was aborted by a guard).
82    pub iterations: usize,
83    /// Wall-clock time spent on this task, in milliseconds.
84    pub latency_ms: u64,
85    /// Optional human-readable detail (e.g. which expected call was missed),
86    /// surfaced in the leaderboard drill-down.
87    #[serde(default)]
88    pub detail: Option<String>,
89}
90
91/// Result of evaluating one candidate's sampling settings against the full
92/// (or pre-screen) task suite.
93#[derive(Debug, Clone, Serialize, Deserialize)]
94pub struct TuneCandidateResult {
95    /// The candidate's resolved sampling settings.
96    pub config: InferenceConfig,
97    /// Where this candidate's settings came from.
98    pub source: CandidateSource,
99    /// Per-task results for this candidate.
100    pub task_results: Vec<TuneTaskResult>,
101    /// Weighted composite score (see [`super::config::ScoreWeights`]).
102    pub composite_score: f64,
103    /// `true` if this candidate was dropped after the pre-screen round and
104    /// never ran the full suite (`task_results` only covers the pre-screen
105    /// tasks in that case).
106    pub pruned: bool,
107    /// Token-generation throughput observed for this candidate, if
108    /// measured (used to normalize the `speed` scoring component).
109    #[serde(default)]
110    pub tg_tps: Option<f64>,
111}
gglib_core/domain/benchmark/tune/result.rs

gglib_core/domain/benchmark/tune/
result.rs
