gglib_core/domain/
admission.rs

1//! What the admission queue and the VRAM resident set look like right now.
2//!
3//! The serializable half of admission control. `gglib-runtime` owns the live
4//! state and the scheduling decisions; this module owns the shape those
5//! decisions are reported in, so `gglib-proxy` can put them on
6//! `GET /v1/proxy/status` without depending on the runtime crate.
7//!
8//! Everything here is a point-in-time projection, the same way the proxy's
9//! active-connection registry projects itself for the dashboard. Nothing here
10//! is authoritative; reading a stale snapshot is always safe.
11//!
12//! ## Why the reasons are carried, not just the numbers
13//!
14//! A user looking at an empty second slot on a card with 12 GB free needs to be
15//! told *why*. So [`SecondarySlotStatus`] carries a stable label for styling
16//! and a ready-to-render sentence, in the same shape
17//! [`CacheRamHealth`](crate::domain::CacheRamHealth) established for the prompt
18//! cache. Consumers branch on the label and print the detail; they never parse
19//! the prose.
20
21use serde::Serialize;
22
23use crate::domain::residency::SecondarySlotDecision;
24
25/// One model resident in VRAM.
26#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
27pub struct ResidentSlotSnapshot {
28    /// Slot index. `0` is the primary.
29    pub slot: usize,
30    /// Name of the resident model.
31    pub model_name: String,
32    /// Database id of the resident model.
33    pub model_id: u32,
34    /// Port its llama-server is listening on.
35    pub port: u16,
36    /// Requests currently holding a lease on this slot.
37    ///
38    /// A slot serving anything at all can never be evicted — a swap must not
39    /// preempt a live generation.
40    pub inflight: u32,
41    /// Whether this is the primary slot, i.e. the one chat traffic and the
42    /// llama.cpp `/slots` poller follow.
43    pub is_primary: bool,
44    /// Seconds this model has been resident.
45    pub resident_for_secs: u64,
46}
47
48/// Requests waiting for one model that is not currently resident.
49#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
50pub struct QueuedModelSnapshot {
51    /// The model they are waiting for.
52    pub model_name: String,
53    /// How many requests are queued.
54    pub waiting: usize,
55    /// Age of the oldest waiter, in milliseconds. A figure that keeps climbing
56    /// is the sign of a model that never goes idle long enough to be swapped
57    /// out — see the runtime's `admission` module for why that is bounded by a
58    /// deadline rather than by preemption.
59    pub oldest_wait_ms: u64,
60}
61
62/// Why the second VRAM slot is or is not in use.
63#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
64pub struct SecondarySlotStatus {
65    /// Stable machine-readable label, for styling. One of `resident`,
66    /// `available`, `too_large`, `no_headroom`, `unknown_footprint`,
67    /// `unknown_budget`.
68    pub state: &'static str,
69    /// Ready-to-render explanation. Phrased for display rather than parsing —
70    /// consumers branch on [`Self::state`].
71    pub detail: String,
72}
73
74impl Default for SecondarySlotStatus {
75    fn default() -> Self {
76        Self {
77            state: "available",
78            detail: "No second model has been requested yet.".to_string(),
79        }
80    }
81}
82
83impl SecondarySlotStatus {
84    /// The status for a slot that currently holds `model_name`.
85    #[must_use]
86    pub fn resident(model_name: &str) -> Self {
87        Self {
88            state: "resident",
89            detail: format!("{model_name} is co-resident and never waits for a swap."),
90        }
91    }
92
93    /// Render the most recent refusal as a status.
94    ///
95    /// A [`SecondarySlotDecision::Grant`] reaching here means the co-load was
96    /// attempted but has not completed (or failed at spawn); it reports as
97    /// available rather than resident, since nothing is loaded.
98    #[must_use]
99    pub fn from_decision(decision: SecondarySlotDecision) -> Self {
100        let detail = match decision {
101            SecondarySlotDecision::Grant { .. } => {
102                "A second model fits and is being loaded.".to_string()
103            }
104            SecondarySlotDecision::RefuseTooLarge {
105                footprint_bytes,
106                ceiling_bytes,
107            } => format!(
108                "The requested model needs about {} — too large for the second slot, which is \
109                 capped at {}. It will be swapped in instead.",
110                format_bytes(footprint_bytes),
111                format_bytes(ceiling_bytes),
112            ),
113            SecondarySlotDecision::RefuseNoHeadroom {
114                footprint_bytes,
115                free_bytes,
116            } => format!(
117                "Not enough free VRAM to keep a second model loaded: it needs about {}, and only \
118                 {} is free.",
119                format_bytes(footprint_bytes),
120                format_bytes(free_bytes),
121            ),
122            SecondarySlotDecision::RefuseUnknownFootprint => {
123                "The requested model's memory footprint could not be estimated from its GGUF \
124                 metadata, so it is swapped in rather than co-loaded."
125                    .to_string()
126            }
127            SecondarySlotDecision::RefuseUnknownBudget => {
128                "gglib cannot read this machine's free VRAM, so it keeps one model loaded at a \
129                 time. Free-VRAM readings are available on NVIDIA and Apple Silicon."
130                    .to_string()
131            }
132        };
133
134        // A grant that has not finished loading holds nothing, so it reports as
135        // available rather than borrowing the decision's own label. Every
136        // refusal keeps its label verbatim, so styling can branch on it.
137        let state = if decision.is_grant() {
138            "available"
139        } else {
140            decision.label()
141        };
142
143        Self { state, detail }
144    }
145}
146
147/// Everything the admission queue and resident set look like right now.
148#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
149pub struct AdmissionSnapshot {
150    /// Models resident in VRAM, primary first. Empty before anything has been
151    /// launched.
152    pub slots: Vec<ResidentSlotSnapshot>,
153    /// Models with requests waiting, oldest waiter first. Empty in the steady
154    /// state — a non-empty list means traffic is being batched behind a swap.
155    pub queued: Vec<QueuedModelSnapshot>,
156    /// Requests that have waited in the queue since the runtime started.
157    /// Compare against [`Self::total_swaps`]: a large ratio is the queue doing
158    /// its job, batching many requests behind one swap.
159    pub total_queued: u64,
160    /// Model swaps performed since the runtime started.
161    pub total_swaps: u64,
162    /// Why the second slot is or is not in use.
163    pub secondary_slot: SecondarySlotStatus,
164}
165
166impl AdmissionSnapshot {
167    /// Total requests waiting across every model.
168    #[must_use]
169    pub fn waiting(&self) -> usize {
170        self.queued.iter().map(|q| q.waiting).sum()
171    }
172
173    /// Total requests currently being served across every resident slot.
174    #[must_use]
175    pub fn inflight(&self) -> u32 {
176        self.slots.iter().map(|s| s.inflight).sum()
177    }
178}
179
180/// Render a byte count the way the launch banner does — one decimal, GiB above
181/// a gibibyte and MiB below, so "307 MiB" does not print as "0.3 GiB".
182fn format_bytes(bytes: u64) -> String {
183    const MIB: f64 = 1024.0 * 1024.0;
184    const GIB: f64 = MIB * 1024.0;
185    #[allow(clippy::cast_precision_loss)]
186    let value = bytes as f64;
187    if value >= GIB {
188        format!("{:.1} GiB", value / GIB)
189    } else {
190        format!("{:.0} MiB", value / MIB)
191    }
192}
193
194#[cfg(test)]
195mod tests {
196    use super::*;
197
198    const MIB: u64 = 1024 * 1024;
199    const GIB: u64 = 1024 * MIB;
200
201    fn slot(name: &str, inflight: u32, is_primary: bool) -> ResidentSlotSnapshot {
202        ResidentSlotSnapshot {
203            slot: usize::from(!is_primary),
204            model_name: name.to_string(),
205            model_id: 1,
206            port: 8080,
207            inflight,
208            is_primary,
209            resident_for_secs: 10,
210        }
211    }
212
213    #[test]
214    fn an_empty_snapshot_reports_nothing_waiting_and_nothing_running() {
215        let snapshot = AdmissionSnapshot::default();
216        assert_eq!(snapshot.waiting(), 0);
217        assert_eq!(snapshot.inflight(), 0);
218        assert_eq!(snapshot.secondary_slot.state, "available");
219    }
220
221    #[test]
222    fn totals_aggregate_across_slots_and_queues() {
223        let snapshot = AdmissionSnapshot {
224            slots: vec![slot("qwen-coder", 2, true), slot("nomic-embed", 1, false)],
225            queued: vec![QueuedModelSnapshot {
226                model_name: "llama-3".to_string(),
227                waiting: 4,
228                oldest_wait_ms: 900,
229            }],
230            total_queued: 12,
231            total_swaps: 2,
232            secondary_slot: SecondarySlotStatus::resident("nomic-embed"),
233        };
234
235        assert_eq!(snapshot.inflight(), 3);
236        assert_eq!(snapshot.waiting(), 4);
237    }
238
239    /// The dashboard has to explain an idle second slot on a card that plainly
240    /// has room, so every refusal names its own reason.
241    #[test]
242    fn each_refusal_carries_a_distinct_label_and_a_populated_detail() {
243        let cases = [
244            SecondarySlotDecision::RefuseTooLarge {
245                footprint_bytes: 10 * GIB,
246                ceiling_bytes: 2 * GIB,
247            },
248            SecondarySlotDecision::RefuseNoHeadroom {
249                footprint_bytes: 307 * MIB,
250                free_bytes: 200 * MIB,
251            },
252            SecondarySlotDecision::RefuseUnknownFootprint,
253            SecondarySlotDecision::RefuseUnknownBudget,
254        ];
255
256        let mut seen = Vec::new();
257        for case in cases {
258            let status = SecondarySlotStatus::from_decision(case);
259            assert!(!status.detail.is_empty(), "{case:?} produced no detail");
260            assert!(!seen.contains(&status.state), "duplicate label {case:?}");
261            seen.push(status.state);
262        }
263    }
264
265    /// The figures a refusal quotes must appear in the sentence it renders —
266    /// "not enough VRAM" without a number is not actionable.
267    #[test]
268    fn a_headroom_refusal_names_both_figures() {
269        let status = SecondarySlotStatus::from_decision(SecondarySlotDecision::RefuseNoHeadroom {
270            footprint_bytes: 307 * MIB,
271            free_bytes: 200 * MIB,
272        });
273
274        assert!(status.detail.contains("307 MiB"), "{}", status.detail);
275        assert!(status.detail.contains("200 MiB"), "{}", status.detail);
276    }
277
278    #[test]
279    fn resident_status_names_the_model() {
280        let status = SecondarySlotStatus::resident("nomic-embed-text");
281        assert_eq!(status.state, "resident");
282        assert!(status.detail.contains("nomic-embed-text"));
283    }
284
285    #[test]
286    fn snapshot_always_serializes() {
287        let json = serde_json::to_string(&AdmissionSnapshot::default())
288            .expect("AdmissionSnapshot must always serialize");
289        assert!(json.contains("secondary_slot"));
290        assert!(json.contains("total_swaps"));
291    }
292
293    // ── format_bytes ─────────────────────────────────────────────────────
294
295    #[test]
296    fn bytes_render_as_gib_above_a_gibibyte_and_mib_below() {
297        assert_eq!(format_bytes(307 * MIB), "307 MiB");
298        assert_eq!(format_bytes(2 * GIB), "2.0 GiB");
299        assert_eq!(format_bytes(GIB + 512 * MIB), "1.5 GiB");
300    }
301}
gglib_core/domain/admission.rs

gglib_core/domain/
admission.rs
