gglib_core/domain/defect_counts.rs
1//! The per-model defect counts — the data half of [`super::defects`].
2//!
3//! [`super::defects`] draws a line between a writer that never interprets and a
4//! reader that never guesses. This is the reader's side of it: one plain struct
5//! of cumulative counts, serialised to the dashboard as it stands. Nearly all of
6//! this file is field documentation, because what a counter does *not* count is
7//! what a person reading a number needs to know, and it has to be said where the
8//! field is declared or it is said nowhere. Beside it, [`LoopGuardTrip`]: the
9//! one fact a writer hands over with a count, which detector raised a trip.
10//!
11//! The ledger that bumps these lives in [`super::defects`], which re-exports
12//! both, so `gglib_core::domain::defects::ModelDefectCounts` still names it.
13
14/// Which of the loop guard's two detectors raised a trip.
15///
16/// The guard is two detectors behind one verdict, and until this existed their
17/// trips went into one number, so nobody could ask whether *stagnation* trips
18/// had become rare, which is the question that decides whether the proxy
19/// keeps `StagnationDetector` in its guard (ADR 0011's first kill criterion,
20/// #947; retiring the detector itself also needs the agent path's reading,
21/// #1091).
22/// Since #1052 a trip is an intervention rather than a rejection: the default
23/// forwards the request with a note.
24///
25/// It says which detector, and nothing about which path. Only the proxy's
26/// pre-dispatch scan records a trip at all: the agent loop runs the same two
27/// detectors and its trips reach no counter (#1091).
28#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
29#[serde(rename_all = "snake_case")]
30#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
31pub enum LoopGuardTrip {
32 /// The same tool-call batch repeated back to back and kept getting the
33 /// same answer.
34 ///
35 /// Also a batch that is not read-only carried past the read-only allowance
36 /// by changing answers. The guard's verdict does not tell those two apart,
37 /// because the remedy is the same, so neither does this.
38 Loop,
39 /// The same assistant text repeated beyond the threshold.
40 Stagnation,
41}
42
43/// Cumulative defect counts for one model.
44#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize)]
45#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
46pub struct ModelDefectCounts {
47 /// Requests the proxy forwarded (or would have, but for a guard) for
48 /// this model — every rate's denominator.
49 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
50 pub requests: u64,
51 /// Requests the loop/stagnation guard acted on.
52 ///
53 /// Since #1052 that is *not* the same as rejected: the guard's default
54 /// forwards a tripped request with a note, and only
55 /// `--loop-guard-mode refuse` rejects it before dispatch. Both count
56 /// here, so this number is a count of **interventions** — per process,
57 /// reset when the daemon restarts. ADR 0011's kill criterion reads the
58 /// loop guard's log instead (`gglib proxy trips`), which outlives the
59 /// process and counts *decisions* rather than snapshots: a noted request
60 /// the embedding check or admission then refuses is a decision there and
61 /// no snapshot here, so the log can count more than this for the same
62 /// traffic.
63 ///
64 /// The sum of the two counts below, kept because it is the row people
65 /// already read and the one an older dashboard knows. Adding all three
66 /// double-counts.
67 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
68 pub loop_guard_trips: u64,
69 /// Of those, the ones [`LoopGuardTrip::Loop`] raised.
70 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
71 pub loop_guard_loops: u64,
72 /// Of those, the ones [`LoopGuardTrip::Stagnation`] raised.
73 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
74 pub loop_guard_stagnations: u64,
75 /// Turns whose tool call failed schema validation and was re-issued,
76 /// with `tool_choice: "required"` or as a second draw under gglib's grammar.
77 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
78 pub repairs_attempted: u64,
79 /// Of those, the re-issues that produced a conformant call.
80 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
81 pub repairs_succeeded: u64,
82 /// Streaming turns that died on an *upstream* mid-stream failure — an
83 /// error event the model server emitted mid-generation, or the byte
84 /// stream itself breaking.
85 ///
86 /// The catastrophic sibling of the repair signal. Both of the counters
87 /// above require a model coherent enough to produce structured output:
88 /// one counts verbatim repetition, the other a tool call that was
89 /// attempted and malformed. A model whose sampling has collapsed
90 /// produces neither — it emits output so far outside the expected shape
91 /// that the model server kills the stream, and the person's turn simply
92 /// fails, invisibly to every other counter here.
93 ///
94 /// Client disconnects are deliberately not in here: hanging up is a
95 /// person's action, not a model defect.
96 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
97 pub stream_errors: u64,
98 /// Turns the model server cut off at the token ceiling
99 /// (`finish_reason == "length"`).
100 ///
101 /// Not a model defect in the same sense as the others — a long answer is
102 /// allowed to be long — but a *rising* rate is how a runaway generation
103 /// looks before anything else notices, and it is the cheapest evidence
104 /// that a context budget is mis-sized.
105 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
106 pub truncated_generations: u64,
107 /// Turns that produced nothing a client can render.
108 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
109 pub empty_responses: u64,
110 /// Of those, the ones that produced reasoning and nothing else.
111 ///
112 /// Counted inside [`Self::empty_responses`] rather than beside it: the
113 /// turn was empty from the client's point of view either way, and the
114 /// distinction is *why*. A model stranding its whole answer in
115 /// `reasoning_content` is a prompt/template problem; one producing
116 /// nothing at all is not.
117 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
118 pub reasoning_only: u64,
119 /// Turns where dialect markup survived normalization into client-visible
120 /// output — the drift alarm, per model rather than fleet-wide.
121 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
122 pub dialect_residue: u64,
123 /// Turns whose tool call could not be validated at all, so repair never
124 /// had an opinion to act on.
125 ///
126 /// The blind spot this makes visible: a client whose tools all use
127 /// `anyOf` gets zero repair coverage *and*, until now, zero evidence of
128 /// that fact. A high rate here means the repair rate below it is
129 /// measuring a much smaller slice of traffic than it appears to.
130 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
131 pub unvalidatable_schemas: u64,
132 /// Turns whose normalization discarded a malformed dialect tool call and
133 /// surfaced the raw body as visible text instead.
134 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
135 pub normalization_errors: u64,
136 /// Turns whose newest tool-call batch repeated the batch before it and
137 /// got an equal result back.
138 ///
139 /// The comparison is against the *preceding* occurrence of that signature,
140 /// not any earlier one: a call that returned A, then B, then A again is
141 /// not counted, because the model did get a different answer last time.
142 ///
143 /// The odd one out, deliberately. Every counter above measures a gglib
144 /// organ firing or a defect in the shape of the model's own output. This
145 /// one measures a condition in the *conversation*: the model asked for
146 /// the same thing twice and the environment answered the same way twice,
147 /// which is the only evidence available that a repeat was genuinely
148 /// stuck rather than progress that happens to look alike.
149 ///
150 /// One increment per turn, like every counter above it — not a tally over
151 /// the replayed history. A client resends the whole conversation each
152 /// turn, so counting history-wide would re-count the same event on every
153 /// later request and grow with the square of session length.
154 ///
155 /// "Equal" means equal after hashing the result's `content` as it
156 /// arrived, per turn. Bounded to the calls the batch actually made, and
157 /// only when every one of them was answered.
158 ///
159 /// Counted whether or not the guard trips — a repeat under the threshold
160 /// is exactly the case a verdict cannot see. Nothing acts on it: it
161 /// exists to answer whether a corrective arm on the input plane would
162 /// ever have a trigger, before one is built.
163 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
164 pub identical_result_repeats: u64,
165 /// Turns whose newest tool-call batch repeated the batch before it but
166 /// whose results could **not** be compared.
167 ///
168 /// The denominator for the counter above, and the reason a zero there can
169 /// be read at all. A repeat gglib could not evaluate is not a repeat that
170 /// did not happen: without this, an instrument that never managed to join
171 /// a single result would look exactly like a fleet with nothing wrong.
172 ///
173 /// Bumps when a client omits `id` on replayed tool calls, when results are
174 /// not contiguous after the assistant turn, or when a parallel batch went
175 /// partly unanswered.
176 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
177 pub repeats_not_evaluated: u64,
178 /// Turns the loop guard would have acted on for repeating, and did not,
179 /// because the answer had moved. A repeat inside the allowance is not one.
180 ///
181 /// Unlike the two above, this is not a fact about the conversation — it is
182 /// a fact about gglib's own reflex, which is what the ledger was chartered
183 /// for before ADR 0006 had to widen it. It reads the detector's run-scoped
184 /// outcome, not the session-wide map those two are computed from, so it is
185 /// a third instrument rather than a third view of one.
186 ///
187 /// It exists because ADR 0010 promoted the results join from an
188 /// observation to a policy input, and a kill criterion nobody can read is
189 /// not a kill criterion. If this dwarfs `identical_result_repeats` in real
190 /// use, the join is being defeated by output that carries a clock rather
191 /// than measuring progress, and the rescue wants narrowing or removing.
192 #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
193 pub repeats_rescued: u64,
194}