gglib_core/domain/
admission.rs

1//! What the admission queue and the VRAM resident set look like right now.
2//!
3//! The serializable half of admission control. `gglib-runtime` owns the live
4//! state and the scheduling decisions; this module owns the shape those
5//! decisions are reported in, so `gglib-proxy` can put them on
6//! `GET /v1/proxy/status` without depending on the runtime crate.
7//!
8//! Everything here is a point-in-time projection, the same way the proxy's
9//! active-connection registry projects itself for the dashboard. Nothing here
10//! is authoritative; reading a stale snapshot is always safe.
11//!
12//! ## Why the reasons are carried, not just the numbers
13//!
14//! A user looking at an empty second slot on a card with 12 GB free needs to be
15//! told *why*. So [`SecondarySlotStatus`] carries a stable label for styling
16//! and a ready-to-render sentence, in the same shape
17//! [`CacheRamHealth`](crate::domain::CacheRamHealth) established for the prompt
18//! cache. Consumers branch on the label and print the detail; they never parse
19//! the prose.
20
21use serde::Serialize;
22
23use crate::domain::residency::SecondarySlotDecision;
24
25/// One model resident in VRAM.
26#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
27#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
28pub struct ResidentSlotSnapshot {
29    /// Slot index. `0` is the primary.
30    pub slot: usize,
31    /// Name of the resident model.
32    pub model_name: String,
33    /// Database id of the resident model.
34    pub model_id: u32,
35    /// Port its llama-server is listening on.
36    pub port: u16,
37    /// Requests currently holding a lease on this slot.
38    ///
39    /// A slot serving anything at all can never be evicted — a swap must not
40    /// preempt a live generation.
41    pub inflight: u32,
42    /// Whether this is the primary slot, i.e. the one chat traffic and the
43    /// llama.cpp `/slots` poller follow.
44    pub is_primary: bool,
45    /// Seconds this model has been resident.
46    #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
47    pub resident_for_secs: u64,
48}
49
50/// Requests waiting for one model that is not currently resident.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
52#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
53pub struct QueuedModelSnapshot {
54    /// The model they are waiting for.
55    pub model_name: String,
56    /// How many requests are queued.
57    pub waiting: usize,
58    /// Age of the oldest waiter, in milliseconds. A figure that keeps climbing
59    /// is the sign of a model that never goes idle long enough to be swapped
60    /// out — see the runtime's `admission` module for why that is bounded by a
61    /// deadline rather than by preemption.
62    #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
63    pub oldest_wait_ms: u64,
64}
65
66/// Why the second VRAM slot is or is not in use.
67#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
68#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
69pub struct SecondarySlotStatus {
70    /// Stable machine-readable label, for styling. One of `resident`,
71    /// `available`, `too_large`, `no_headroom`, `unknown_footprint`,
72    /// `unknown_budget`.
73    ///
74    /// A `&'static str`, which ts-rs would render as a bare `string` — losing
75    /// the exhaustiveness the GUI's icon table and tone function are written
76    /// against. The override restates the closed set above, the same way
77    /// `gglib_proxy::dashboard::CacheStatus::ram_state` does.
78    ///
79    /// Three places set it, and all three must stay inside that set:
80    /// [`Self::default`] and [`Self::resident`] each write one literal, and
81    /// [`Self::from_decision`] takes `SecondarySlotDecision::label` for every
82    /// refusal. `label` also answers `"grant"`, which never reaches here —
83    /// `from_decision` maps a grant to `"available"` before consulting it.
84    #[cfg_attr(
85        feature = "ts-bindings",
86        ts(type = "\"resident\" | \"available\" | \"too_large\" \
87                   | \"no_headroom\" | \"unknown_footprint\" | \"unknown_budget\"")
88    )]
89    pub state: &'static str,
90    /// Ready-to-render explanation. Phrased for display rather than parsing —
91    /// consumers branch on [`Self::state`].
92    pub detail: String,
93}
94
95impl Default for SecondarySlotStatus {
96    fn default() -> Self {
97        Self {
98            state: "available",
99            detail: "No second model has been requested yet.".to_string(),
100        }
101    }
102}
103
104impl SecondarySlotStatus {
105    /// The status for a slot that currently holds `model_name`.
106    #[must_use]
107    pub fn resident(model_name: &str) -> Self {
108        Self {
109            state: "resident",
110            detail: format!("{model_name} is co-resident and never waits for a swap."),
111        }
112    }
113
114    /// Render the most recent refusal as a status.
115    ///
116    /// A [`SecondarySlotDecision::Grant`] reaching here means the co-load was
117    /// attempted but has not completed (or failed at spawn); it reports as
118    /// available rather than resident, since nothing is loaded.
119    #[must_use]
120    pub fn from_decision(decision: SecondarySlotDecision) -> Self {
121        let detail = match decision {
122            SecondarySlotDecision::Grant { .. } => {
123                "A second model fits and is being loaded.".to_string()
124            }
125            SecondarySlotDecision::RefuseTooLarge {
126                footprint_bytes,
127                ceiling_bytes,
128            } => format!(
129                "The requested model needs about {} — too large for the second slot, which is \
130                 capped at {}. It will be swapped in instead.",
131                format_bytes(footprint_bytes),
132                format_bytes(ceiling_bytes),
133            ),
134            SecondarySlotDecision::RefuseNoHeadroom {
135                footprint_bytes,
136                free_bytes,
137            } => format!(
138                "Not enough free VRAM to keep a second model loaded: it needs about {}, and only \
139                 {} is free.",
140                format_bytes(footprint_bytes),
141                format_bytes(free_bytes),
142            ),
143            SecondarySlotDecision::RefuseUnknownFootprint => {
144                "The requested model's memory footprint could not be estimated from its GGUF \
145                 metadata, so it is swapped in rather than co-loaded."
146                    .to_string()
147            }
148            SecondarySlotDecision::RefuseUnknownBudget => {
149                "gglib cannot read this machine's free VRAM, so it keeps one model loaded at a \
150                 time. Free-VRAM readings are available on NVIDIA and Apple Silicon."
151                    .to_string()
152            }
153        };
154
155        // A grant that has not finished loading holds nothing, so it reports as
156        // available rather than borrowing the decision's own label. Every
157        // refusal keeps its label verbatim, so styling can branch on it.
158        let state = if decision.is_grant() {
159            "available"
160        } else {
161            decision.label()
162        };
163
164        Self { state, detail }
165    }
166}
167
168/// Everything the admission queue and resident set look like right now.
169#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
170#[cfg_attr(feature = "ts-bindings", derive(ts_rs::TS), ts(export))]
171pub struct AdmissionSnapshot {
172    /// Models resident in VRAM, primary first. Empty before anything has been
173    /// launched.
174    pub slots: Vec<ResidentSlotSnapshot>,
175    /// Models with requests waiting, oldest waiter first. Empty in the steady
176    /// state — a non-empty list means traffic is being batched behind a swap.
177    pub queued: Vec<QueuedModelSnapshot>,
178    /// Requests that have waited in the queue since the runtime started.
179    /// Compare against [`Self::total_swaps`]: a large ratio is the queue doing
180    /// its job, batching many requests behind one swap.
181    #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
182    pub total_queued: u64,
183    /// Model swaps performed since the runtime started.
184    #[cfg_attr(feature = "ts-bindings", ts(type = "number"))]
185    pub total_swaps: u64,
186    /// Why the second slot is or is not in use.
187    pub secondary_slot: SecondarySlotStatus,
188}
189
190impl AdmissionSnapshot {
191    /// Total requests waiting across every model.
192    #[must_use]
193    pub fn waiting(&self) -> usize {
194        self.queued.iter().map(|q| q.waiting).sum()
195    }
196
197    /// Total requests currently being served across every resident slot.
198    #[must_use]
199    pub fn inflight(&self) -> u32 {
200        self.slots.iter().map(|s| s.inflight).sum()
201    }
202}
203
204/// Render a byte count the way the launch banner does — one decimal, GiB above
205/// a gibibyte and MiB below, so "307 MiB" does not print as "0.3 GiB".
206fn format_bytes(bytes: u64) -> String {
207    const MIB: f64 = 1024.0 * 1024.0;
208    const GIB: f64 = MIB * 1024.0;
209    #[allow(clippy::cast_precision_loss)]
210    let value = bytes as f64;
211    if value >= GIB {
212        format!("{:.1} GiB", value / GIB)
213    } else {
214        format!("{:.0} MiB", value / MIB)
215    }
216}
217
218#[cfg(test)]
219mod tests {
220    use super::*;
221
222    const MIB: u64 = 1024 * 1024;
223    const GIB: u64 = 1024 * MIB;
224
225    fn slot(name: &str, inflight: u32, is_primary: bool) -> ResidentSlotSnapshot {
226        ResidentSlotSnapshot {
227            slot: usize::from(!is_primary),
228            model_name: name.to_string(),
229            model_id: 1,
230            port: 8080,
231            inflight,
232            is_primary,
233            resident_for_secs: 10,
234        }
235    }
236
237    #[test]
238    fn an_empty_snapshot_reports_nothing_waiting_and_nothing_running() {
239        let snapshot = AdmissionSnapshot::default();
240        assert_eq!(snapshot.waiting(), 0);
241        assert_eq!(snapshot.inflight(), 0);
242        assert_eq!(snapshot.secondary_slot.state, "available");
243    }
244
245    #[test]
246    fn totals_aggregate_across_slots_and_queues() {
247        let snapshot = AdmissionSnapshot {
248            slots: vec![slot("qwen-coder", 2, true), slot("nomic-embed", 1, false)],
249            queued: vec![QueuedModelSnapshot {
250                model_name: "llama-3".to_string(),
251                waiting: 4,
252                oldest_wait_ms: 900,
253            }],
254            total_queued: 12,
255            total_swaps: 2,
256            secondary_slot: SecondarySlotStatus::resident("nomic-embed"),
257        };
258
259        assert_eq!(snapshot.inflight(), 3);
260        assert_eq!(snapshot.waiting(), 4);
261    }
262
263    /// The dashboard has to explain an idle second slot on a card that plainly
264    /// has room, so every refusal names its own reason.
265    #[test]
266    fn each_refusal_carries_a_distinct_label_and_a_populated_detail() {
267        let cases = [
268            SecondarySlotDecision::RefuseTooLarge {
269                footprint_bytes: 10 * GIB,
270                ceiling_bytes: 2 * GIB,
271            },
272            SecondarySlotDecision::RefuseNoHeadroom {
273                footprint_bytes: 307 * MIB,
274                free_bytes: 200 * MIB,
275            },
276            SecondarySlotDecision::RefuseUnknownFootprint,
277            SecondarySlotDecision::RefuseUnknownBudget,
278        ];
279
280        let mut seen = Vec::new();
281        for case in cases {
282            let status = SecondarySlotStatus::from_decision(case);
283            assert!(!status.detail.is_empty(), "{case:?} produced no detail");
284            assert!(!seen.contains(&status.state), "duplicate label {case:?}");
285            seen.push(status.state);
286        }
287    }
288
289    /// The figures a refusal quotes must appear in the sentence it renders —
290    /// "not enough VRAM" without a number is not actionable.
291    #[test]
292    fn a_headroom_refusal_names_both_figures() {
293        let status = SecondarySlotStatus::from_decision(SecondarySlotDecision::RefuseNoHeadroom {
294            footprint_bytes: 307 * MIB,
295            free_bytes: 200 * MIB,
296        });
297
298        assert!(status.detail.contains("307 MiB"), "{}", status.detail);
299        assert!(status.detail.contains("200 MiB"), "{}", status.detail);
300    }
301
302    #[test]
303    fn resident_status_names_the_model() {
304        let status = SecondarySlotStatus::resident("nomic-embed-text");
305        assert_eq!(status.state, "resident");
306        assert!(status.detail.contains("nomic-embed-text"));
307    }
308
309    #[test]
310    fn snapshot_always_serializes() {
311        let json = serde_json::to_string(&AdmissionSnapshot::default())
312            .expect("AdmissionSnapshot must always serialize");
313        assert!(json.contains("secondary_slot"));
314        assert!(json.contains("total_swaps"));
315    }
316
317    // ── format_bytes ─────────────────────────────────────────────────────
318
319    #[test]
320    fn bytes_render_as_gib_above_a_gibibyte_and_mib_below() {
321        assert_eq!(format_bytes(307 * MIB), "307 MiB");
322        assert_eq!(format_bytes(2 * GIB), "2.0 GiB");
323        assert_eq!(format_bytes(GIB + 512 * MIB), "1.5 GiB");
324    }
325}
gglib_core/domain/admission.rs

gglib_core/domain/
admission.rs
