gglib_core/
server_config.rs

1//! Canonical context-size resolver (4-level fallback chain).
2//!
3//! Extracted to `gglib-core` so that crates which cannot depend on
4//! `gglib-runtime` (e.g. `gglib-proxy`) can still use the same resolution
5//! logic for idle-model advertisements in `/v1/models`.
6
7use anyhow::{Result, anyhow};
8use std::path::PathBuf;
9
10use crate::domain::InferenceConfig;
11use crate::settings::DEFAULT_CONTEXT_SIZE;
12
13// =============================================================================
14// CLI flag parsing (deferred resolution)
15// =============================================================================
16
17/// A parsed `--ctx-size` CLI flag, before it is resolved against model
18/// metadata.
19///
20/// CLI argument parsing happens before the model is fetched from the
21/// database, so the raw flag cannot be resolved to a concrete value at parse
22/// time. [`CtxSizeArg::parse`] only validates the *shape* of the flag
23/// (numeric or the literal `max`); callers must call [`CtxSizeArg::resolve`]
24/// once the model (and its GGUF `context_length`) is available.
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum CtxSizeArg {
27    /// User passed `max` — resolve against the model's GGUF context length.
28    Max,
29    /// User passed an explicit numeric value.
30    Value(u64),
31}
32
33impl CtxSizeArg {
34    /// Parse a raw `--ctx-size` flag value.
35    ///
36    /// Accepts a positive integer or the case-insensitive literal `max`.
37    /// Anything else is a hard error — invalid input must never be
38    /// silently ignored.
39    pub fn parse(raw: &str) -> Result<Self> {
40        let trimmed = raw.trim();
41        if trimmed.eq_ignore_ascii_case("max") {
42            return Ok(Self::Max);
43        }
44        trimmed.parse::<u64>().map(Self::Value).map_err(|_| {
45            anyhow!("Invalid context size '{trimmed}'. Use a positive number or 'max'")
46        })
47    }
48
49    /// Resolve this flag into a concrete context size, now that the model's
50    /// GGUF metadata is available.
51    ///
52    /// - `Max` resolves to `model_max_ctx` (`None` if the model has no
53    ///   recorded context length — falls through to the next tier).
54    /// - `Value(n)` always resolves to `Some(n)`.
55    pub const fn resolve(self, model_max_ctx: Option<u64>) -> Option<u64> {
56        match self {
57            Self::Max => model_max_ctx,
58            Self::Value(v) => Some(v),
59        }
60    }
61}
62
63/// Parse an optional raw `--ctx-size` flag into a [`CtxSizeArg`].
64///
65/// Convenience wrapper for CLI call sites: `None` (flag omitted) stays
66/// `None`; `Some(raw)` is parsed and propagates a hard error on invalid
67/// input via `?`.
68pub fn parse_ctx_size_flag(raw: Option<&str>) -> Result<Option<CtxSizeArg>> {
69    raw.map(CtxSizeArg::parse).transpose()
70}
71
72// =============================================================================
73// Options
74// =============================================================================
75
76/// Caller-supplied overrides for [`resolve_context_size`].
77///
78/// All fields default to `None`, which means "fall through to next tier".
79#[derive(Debug, Clone, Default)]
80pub struct ServerConfigOptions {
81    /// Override the context window size forwarded to llama-server.
82    /// `None` lets llama-server use its built-in default.
83    pub context_size: Option<u64>,
84
85    /// Per-model server defaults context length (from `Model.server_defaults.context_length`).
86    /// Second tier in fallback chain.
87    pub model_server_ctx: Option<usize>,
88
89    /// Global app setting for default context size (from `Settings.default_context_size`).
90    /// Third tier in fallback chain.
91    pub global_default_ctx: Option<u64>,
92
93    /// Bind llama-server to a specific port instead of letting the allocator
94    /// choose.
95    pub port: Option<u16>,
96
97    /// Override Jinja template support.
98    /// - `None` → auto-detect: enabled when the model has the `"agent"` tag.
99    /// - `Some(true)` → force enable regardless of tags.
100    /// - `Some(false)` → force disable regardless of tags.
101    pub jinja: Option<bool>,
102
103    /// Override the reasoning format passed to llama-server.
104    /// - `None` → auto-detect from model tags (e.g. `"reasoning"` tag).
105    /// - `Some("none")` → explicitly suppress reasoning extraction even if the
106    ///   model has a reasoning tag.
107    /// - `Some("deepseek")` / `Some("deepseek-legacy")` → force a specific
108    ///   format.
109    pub reasoning_format: Option<String>,
110
111    /// Override the MTP draft token count.
112    /// - `None` → auto-detect: enabled with default `n=2` when the model has
113    ///   the `"mtp"` tag.
114    /// - `Some(0)` → explicitly disable MTP even if the model has the `"mtp"`
115    ///   tag.
116    /// - `Some(n)` → enable MTP with `n` draft tokens.
117    pub mtp_draft_n_max: Option<u32>,
118
119    /// Override the MTP acceptance probability threshold.
120    /// Only meaningful when MTP is enabled. `None` uses the default (`0.75`).
121    pub mtp_draft_p_min: Option<f32>,
122
123    /// Directory for llama-server KV cache slot persistence (`--slot-save-path`).
124    /// - `None` — disk slot persistence disabled, no `--slot-save-path` flag.
125    /// - `Some(dir)` — enables slot save/restore.
126    ///   Direct pass-through, no tag-based auto-detection (unlike jinja/MTP/reasoning).
127    ///   Independent of `cache_ram_mb`/`cache_reuse` below.
128    pub slot_save_path: Option<PathBuf>,
129
130    /// RAM budget in MiB for llama-server's own host-RAM prompt cache
131    /// (`--cache-ram`). `None` leaves llama-server's built-in default. `Some(0)`
132    /// disables the cache. Direct pass-through, no tag-based auto-detection.
133    pub cache_ram_mb: Option<u64>,
134
135    /// Minimum chunk size in tokens for KV-shift cache reuse past the first
136    /// prefix divergence point (`--cache-reuse`). `None` leaves the feature
137    /// off. Direct pass-through, no tag-based auto-detection.
138    pub cache_reuse: Option<u32>,
139
140    /// Explicit override for the K cache element type (`--cache-type-k`).
141    /// `None` resolves to the `q8_0` default (see
142    /// `gglib_runtime::llama::args::resolve_kv_cache_types`), unless
143    /// `GGLIB_DISABLE_KV_QUANT=1` is set.
144    pub cache_type_k: Option<crate::cache_config::KvCacheType>,
145
146    /// Explicit override for the V cache element type (`--cache-type-v`).
147    /// Same resolution as [`Self::cache_type_k`]. Quantizing V additionally
148    /// requires Flash Attention to be active — see
149    /// `gglib_runtime::llama::args::kv_cache_type` module docs.
150    pub cache_type_v: Option<crate::cache_config::KvCacheType>,
151
152    /// Inference parameter overrides (temperature, top-p, etc.) forwarded
153    /// directly to llama-server.
154    pub inference_params: Option<InferenceConfig>,
155
156    /// Whether to memory-lock the model into RAM (`--mlock`).
157    /// `None` defaults to `false` in `build_server_config()`.
158    pub mlock: Option<bool>,
159}
160
161impl ServerConfigOptions {
162    /// Field-wise merge: every `Some` in `over` wins, every `None` falls
163    /// through to `self`.
164    ///
165    /// This is the single layering primitive behind both places where two sets
166    /// of options meet:
167    ///
168    /// - the 3-tier cascade in `UnifiedServerConfig::resolved_options`, where
169    ///   global defaults are the base and explicit CLI/GUI overrides are `over`;
170    /// - per-call launch overrides layered on top of a `ProcessManager`'s
171    ///   standing template.
172    ///
173    /// Note that this merges *options*, not resolved values — the tier chain
174    /// baked into [`resolve_context_size`] (request → per-model → global →
175    /// hardcoded) still runs afterwards on the merged result, so overlaying
176    /// never collapses those tiers early.
177    ///
178    /// `over` is destructured exhaustively on purpose: adding a field to this
179    /// struct then fails to compile until it is given merge semantics here,
180    /// rather than being silently dropped.
181    #[must_use]
182    pub fn overlay(&self, over: &Self) -> Self {
183        let Self {
184            context_size,
185            model_server_ctx,
186            global_default_ctx,
187            port,
188            jinja,
189            reasoning_format,
190            mtp_draft_n_max,
191            mtp_draft_p_min,
192            slot_save_path,
193            cache_ram_mb,
194            cache_reuse,
195            cache_type_k,
196            cache_type_v,
197            inference_params,
198            mlock,
199        } = over;
200
201        Self {
202            context_size: context_size.or(self.context_size),
203            model_server_ctx: model_server_ctx.or(self.model_server_ctx),
204            global_default_ctx: global_default_ctx.or(self.global_default_ctx),
205            port: port.or(self.port),
206            jinja: jinja.or(self.jinja),
207            reasoning_format: reasoning_format
208                .clone()
209                .or_else(|| self.reasoning_format.clone()),
210            mtp_draft_n_max: mtp_draft_n_max.or(self.mtp_draft_n_max),
211            mtp_draft_p_min: mtp_draft_p_min.or(self.mtp_draft_p_min),
212            slot_save_path: slot_save_path
213                .clone()
214                .or_else(|| self.slot_save_path.clone()),
215            cache_ram_mb: cache_ram_mb.or(self.cache_ram_mb),
216            cache_reuse: cache_reuse.or(self.cache_reuse),
217            cache_type_k: cache_type_k.or(self.cache_type_k),
218            cache_type_v: cache_type_v.or(self.cache_type_v),
219            inference_params: inference_params
220                .clone()
221                .or_else(|| self.inference_params.clone()),
222            mlock: mlock.or(self.mlock),
223        }
224    }
225}
226
227// =============================================================================
228// Resolver
229// =============================================================================
230
231/// Resolve context size using the 4-level fallback chain.
232/// 1. Runtime request / CLI flag (`opts.context_size`) — highest priority
233/// 2. Per-model server defaults (`opts.model_server_ctx`) — from DB
234/// 3. Global app setting (`opts.global_default_ctx`)
235/// 4. Hardcoded default (`DEFAULT_CONTEXT_SIZE` = 4096) — lowest priority
236pub fn resolve_context_size(opts: &ServerConfigOptions) -> u64 {
237    opts.context_size
238        .or_else(|| opts.model_server_ctx.map(|v| v as u64))
239        .or(opts.global_default_ctx)
240        .unwrap_or(DEFAULT_CONTEXT_SIZE)
241}
242
243// =============================================================================
244// Host-RAM prompt cache budget (`--cache-ram`)
245// =============================================================================
246
247// `CacheRamSetting` now lives in `crate::cache_config`, alongside
248// `KvCacheType` — cache-related config resolution has one home. Re-exported
249// here so existing `gglib_core::server_config::CacheRamSetting` call sites
250// keep working.
251pub use crate::cache_config::CacheRamSetting;
252
253// Cache-RAM budget constants and [`compute_auto_cache_ram_mb`] now live in
254// `crate::domain::cache_budget` (re-exported from `crate::domain`), alongside
255// the rest of the domain's pure calculations.
256pub use crate::domain::cache_budget::{
257    CACHE_RAM_FLOOR_BYTES, CACHE_RAM_HEADROOM_BYTES, CACHE_RAM_UNKNOWN_KV_ALLOWANCE_BYTES,
258    compute_auto_cache_ram_mb,
259};
260
261#[cfg(test)]
262mod tests {
263    use crate::server_config::{ServerConfigOptions, resolve_context_size};
264    use crate::settings::DEFAULT_CONTEXT_SIZE;
265
266    #[test]
267    fn test_resolve_context_size_default_when_all_none() {
268        let opts = ServerConfigOptions::default();
269        assert_eq!(resolve_context_size(&opts), DEFAULT_CONTEXT_SIZE);
270    }
271
272    // Cache-RAM budget math tests now live in
273    // `crate::domain::cache_budget::tests`, alongside the function itself.
274    use crate::server_config::CacheRamSetting;
275
276    /// Every launch surface should auto-size unless it opts out, so `Auto`
277    /// has to be the `Default` variant.
278    #[test]
279    fn cache_ram_setting_defaults_to_auto() {
280        assert_eq!(CacheRamSetting::default(), CacheRamSetting::Auto);
281    }
282
283    #[test]
284    fn test_resolve_context_size_global_beats_default() {
285        let opts = ServerConfigOptions {
286            global_default_ctx: Some(8192),
287            ..Default::default()
288        };
289        assert_eq!(resolve_context_size(&opts), 8192);
290    }
291
292    #[test]
293    fn test_resolve_context_size_model_beats_global() {
294        let opts = ServerConfigOptions {
295            model_server_ctx: Some(16_384),
296            global_default_ctx: Some(8192),
297            ..Default::default()
298        };
299        assert_eq!(resolve_context_size(&opts), 16_384);
300    }
301
302    #[test]
303    fn test_resolve_context_size_runtime_beats_all() {
304        let opts = ServerConfigOptions {
305            context_size: Some(32_768),
306            model_server_ctx: Some(16_384),
307            global_default_ctx: Some(8192),
308            ..Default::default()
309        };
310        assert_eq!(resolve_context_size(&opts), 32_768);
311    }
312
313    #[test]
314    fn test_resolve_context_size_model_without_global() {
315        let opts = ServerConfigOptions {
316            model_server_ctx: Some(2048),
317            ..Default::default()
318        };
319        assert_eq!(resolve_context_size(&opts), 2048);
320    }
321
322    #[test]
323    fn test_resolve_context_size_zero_is_valid() {
324        let opts = ServerConfigOptions {
325            context_size: Some(0),
326            ..Default::default()
327        };
328        assert_eq!(resolve_context_size(&opts), 0);
329    }
330
331    // -------------------------------------------------------------------
332    // CtxSizeArg / parse_ctx_size_flag
333    // -------------------------------------------------------------------
334
335    use crate::server_config::{CtxSizeArg, parse_ctx_size_flag};
336
337    #[test]
338    fn ctx_size_arg_parses_explicit_numeric() {
339        assert_eq!(CtxSizeArg::parse("8192").unwrap(), CtxSizeArg::Value(8192));
340    }
341
342    #[test]
343    fn ctx_size_arg_parses_max_case_insensitive() {
344        assert_eq!(CtxSizeArg::parse("max").unwrap(), CtxSizeArg::Max);
345        assert_eq!(CtxSizeArg::parse("MAX").unwrap(), CtxSizeArg::Max);
346        assert_eq!(CtxSizeArg::parse("  Max  ").unwrap(), CtxSizeArg::Max);
347    }
348
349    #[test]
350    fn ctx_size_arg_invalid_string_is_hard_error() {
351        assert!(CtxSizeArg::parse("banana").is_err());
352    }
353
354    #[test]
355    fn ctx_size_arg_max_resolves_to_model_metadata() {
356        assert_eq!(CtxSizeArg::Max.resolve(Some(131_072)), Some(131_072));
357    }
358
359    #[test]
360    fn ctx_size_arg_max_without_model_metadata_resolves_to_none() {
361        assert_eq!(CtxSizeArg::Max.resolve(None), None);
362    }
363
364    #[test]
365    fn ctx_size_arg_value_ignores_model_metadata() {
366        assert_eq!(CtxSizeArg::Value(4096).resolve(Some(131_072)), Some(4096));
367    }
368
369    #[test]
370    fn parse_ctx_size_flag_none_when_flag_omitted() {
371        assert_eq!(parse_ctx_size_flag(None).unwrap(), None);
372    }
373
374    #[test]
375    fn parse_ctx_size_flag_propagates_parse_error() {
376        assert!(parse_ctx_size_flag(Some("not-a-number")).is_err());
377    }
378
379    // -------------------------------------------------------------------
380    // overlay
381    // -------------------------------------------------------------------
382
383    use crate::cache_config::KvCacheType;
384    use crate::domain::InferenceConfig;
385    use std::path::PathBuf;
386
387    /// Every field set, so a merge that drops one is visible. `marker` is a
388    /// `u8` purely so each field can widen losslessly via `From`.
389    fn populated(marker: u8) -> ServerConfigOptions {
390        ServerConfigOptions {
391            context_size: Some(u64::from(marker)),
392            model_server_ctx: Some(usize::from(marker)),
393            global_default_ctx: Some(u64::from(marker)),
394            port: Some(u16::from(marker)),
395            jinja: Some(true),
396            reasoning_format: Some(format!("fmt-{marker}")),
397            mtp_draft_n_max: Some(u32::from(marker)),
398            mtp_draft_p_min: Some(f32::from(marker)),
399            slot_save_path: Some(PathBuf::from(format!("/slots/{marker}"))),
400            cache_ram_mb: Some(u64::from(marker)),
401            cache_reuse: Some(u32::from(marker)),
402            cache_type_k: Some(KvCacheType::Q8_0),
403            cache_type_v: Some(KvCacheType::F16),
404            inference_params: Some(InferenceConfig {
405                temperature: Some(f32::from(marker)),
406                ..Default::default()
407            }),
408            mlock: Some(true),
409        }
410    }
411
412    /// A fully-populated `over` must win on every single field. Compared
413    /// field-by-field rather than wholesale so a failure names the culprit.
414    #[test]
415    fn overlay_over_wins_on_every_field() {
416        let merged = populated(1).overlay(&populated(2));
417
418        assert_eq!(merged.context_size, Some(2));
419        assert_eq!(merged.model_server_ctx, Some(2));
420        assert_eq!(merged.global_default_ctx, Some(2));
421        assert_eq!(merged.port, Some(2));
422        assert_eq!(merged.jinja, Some(true));
423        assert_eq!(merged.reasoning_format.as_deref(), Some("fmt-2"));
424        assert_eq!(merged.mtp_draft_n_max, Some(2));
425        assert_eq!(merged.mtp_draft_p_min, Some(2.0));
426        assert_eq!(merged.slot_save_path, Some(PathBuf::from("/slots/2")));
427        assert_eq!(merged.cache_ram_mb, Some(2));
428        assert_eq!(merged.cache_reuse, Some(2));
429        assert_eq!(merged.cache_type_k, Some(KvCacheType::Q8_0));
430        assert_eq!(merged.cache_type_v, Some(KvCacheType::F16));
431        assert_eq!(
432            merged.inference_params.and_then(|c| c.temperature),
433            Some(2.0)
434        );
435        assert_eq!(merged.mlock, Some(true));
436    }
437
438    /// The direction that actually does the work — and the identity property
439    /// the cascade leans on when a tier has no opinion: a base with values and
440    /// an `over` that is silent must keep every base value.
441    #[test]
442    fn overlay_falls_through_to_base_on_every_field() {
443        let merged = populated(1).overlay(&ServerConfigOptions::default());
444
445        assert_eq!(merged.context_size, Some(1));
446        assert_eq!(merged.model_server_ctx, Some(1));
447        assert_eq!(merged.global_default_ctx, Some(1));
448        assert_eq!(merged.port, Some(1));
449        assert_eq!(merged.jinja, Some(true));
450        assert_eq!(merged.reasoning_format.as_deref(), Some("fmt-1"));
451        assert_eq!(merged.mtp_draft_n_max, Some(1));
452        assert_eq!(merged.mtp_draft_p_min, Some(1.0));
453        assert_eq!(merged.slot_save_path, Some(PathBuf::from("/slots/1")));
454        assert_eq!(merged.cache_ram_mb, Some(1));
455        assert_eq!(merged.cache_reuse, Some(1));
456        assert_eq!(merged.cache_type_k, Some(KvCacheType::Q8_0));
457        assert_eq!(merged.cache_type_v, Some(KvCacheType::F16));
458        assert_eq!(
459            merged.inference_params.and_then(|c| c.temperature),
460            Some(1.0)
461        );
462        assert_eq!(merged.mlock, Some(true));
463    }
464
465    /// Per-field interleaving: neither side wholesale-replaces the other.
466    #[test]
467    fn overlay_merges_per_field_not_wholesale() {
468        let base = ServerConfigOptions {
469            context_size: Some(8192),
470            mlock: Some(true),
471            ..Default::default()
472        };
473        let over = ServerConfigOptions {
474            port: Some(5500),
475            mlock: Some(false),
476            ..Default::default()
477        };
478
479        let merged = base.overlay(&over);
480
481        assert_eq!(merged.context_size, Some(8192), "base-only field survives");
482        assert_eq!(merged.port, Some(5500), "over-only field lands");
483        assert_eq!(merged.mlock, Some(false), "contested field goes to over");
484    }
485
486    /// `Some(false)` is an explicit opinion, not an absence — it has to beat a
487    /// `Some(true)` underneath it. This is what lets `--mtp-draft-n-max 0` and
488    /// an explicit jinja-off override a tag-derived default.
489    #[test]
490    fn overlay_treats_some_false_as_an_override() {
491        let base = ServerConfigOptions {
492            jinja: Some(true),
493            ..Default::default()
494        };
495        let over = ServerConfigOptions {
496            jinja: Some(false),
497            ..Default::default()
498        };
499
500        assert_eq!(base.overlay(&over).jinja, Some(false));
501    }
502}
gglib_core/server_config.rs

gglib_core/
server_config.rs
