gglib_core/ports/process_runner.rs
1//! The two value types a model-server launch is described and tracked by.
2//!
3//! [`ServerConfig`] is what a caller asks for; [`ProcessHandle`] is what it
4//! gets back. The `ProcessRunner` trait this file is named after is gone, and
5//! `ModelRuntimePort` is the port that actually carries launches. The file
6//! keeps its name only because these two types are reached as
7//! `ports::{ServerConfig, ProcessHandle}` regardless.
8//!
9//! It had implementors, contrary to what this comment said until now: four at
10//! `4a6fcf4b^`, including the production `LlamaServerRunner` in
11//! `gglib-runtime/src/runner.rs`. That runner went in #708 and the other three
12//! were test doubles, which is what left the trait with nothing implementing
13//! it by the time #849 removed it — a different and much less interesting
14//! claim than "nothing ever did".
15
16use serde::{Deserialize, Serialize};
17use std::path::PathBuf;
18
19use crate::domain::InferenceConfig;
20
21/// What position a launch takes on Jinja chat templating.
22///
23/// Three states rather than a bool because llama-server's default is jinja
24/// **on**: `use_jinja` initialises to `true` (`common/common.h:621`) and
25/// `common/arg.cpp:1394-1399` flips it off only for the completion and mtmd
26/// examples — never for the server. So "gglib emits no flag" and "gglib turns
27/// jinja off" are two different launches, and a bool could only ever name one
28/// of them. It named the wrong one: `false` meant *emit nothing*, so a user who
29/// explicitly disabled Jinja got a server running with it anyway, silently.
30///
31/// The distinction is in the type rather than in a convention because both
32/// falsy cases are reachable and they must not be conflated — see
33/// [`Self::Defer`].
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
35#[serde(rename_all = "snake_case")]
36pub enum JinjaMode {
37 /// Emit no jinja flag at all and let llama-server decide.
38 ///
39 /// The default, and what an untagged model with no override resolves to.
40 /// Against this pinned llama.cpp that means jinja is **on** — deferring is
41 /// not the same as turning it off, and gglib does not pretend otherwise.
42 #[default]
43 Defer,
44 /// Emit `--jinja`.
45 On,
46 /// Emit `--no-jinja`.
47 ///
48 /// Reached only from an explicit caller override. Nothing tag-derived
49 /// produces this: taking jinja away removes tool-call templating and
50 /// template kwargs, which is a decision only the user gets to make.
51 Off,
52}
53
54/// Configuration for starting a model server.
55///
56/// This is an intent-based configuration — it expresses what the caller
57/// wants, not how the server should be started. All typed fields are
58/// handled by `build_and_spawn()`; `extra_args` is an escape hatch for
59/// flags not yet promoted to first-class fields.
60#[derive(Debug, Clone, Serialize, Deserialize)]
61pub struct ServerConfig {
62 /// Database ID of the model to serve.
63 pub model_id: i64,
64 /// Human-readable model name.
65 pub model_name: String,
66 /// Path to the model file.
67 pub model_path: PathBuf,
68 /// Port to listen on (if None, a free port will be assigned).
69 pub port: Option<u16>,
70 /// Base port for allocation when port is None.
71 pub base_port: u16,
72 /// Context size to use (if None, use model default).
73 pub context_size: Option<u64>,
74 /// Number of GPU layers to offload (if None, use default).
75 pub gpu_layers: Option<i32>,
76 /// What this launch says about Jinja templating for chat formats.
77 ///
78 /// See [`JinjaMode`] — [`JinjaMode::Defer`] emits nothing, which leaves
79 /// llama-server's own (on) default in place rather than turning jinja off.
80 pub jinja: JinjaMode,
81 /// Reasoning format override (e.g., `"deepseek"`, `"none"`).
82 pub reasoning_format: Option<String>,
83 /// Number of MTP draft tokens to speculate ahead (`--spec-draft-n-max`).
84 ///
85 /// `None` means MTP speculative decoding is disabled. When `Some(n)`,
86 /// `--spec-type draft-mtp` and `--spec-draft-n-max n` are passed to
87 /// llama-server. Recommended value: `2` (Unsloth default).
88 pub spec_draft_n_max: Option<u32>,
89 /// Minimum acceptance probability for MTP draft tokens (`--spec-draft-p-min`).
90 ///
91 /// Only meaningful when `spec_draft_n_max` is `Some`. Skipping low-confidence
92 /// draft tokens is especially important on Apple Silicon (Metal) to avoid
93 /// throughput regression. Recommended value: `0.75`.
94 pub spec_draft_p_min: Option<f32>,
95 /// Inference sampling parameters (temperature, `top_p`, etc.).
96 ///
97 /// **Nothing reads this.** ADR 0003 deleted `to_cli_args` and its one
98 /// caller, so no sampler value becomes a command-line argument any more,
99 /// and the launch narration reports sampling from
100 /// `llama::args::sampling`'s constants rather than from here. The field is
101 /// written by `build_server_config` and read by nobody.
102 ///
103 /// Kept for now rather than removed because the plumbing that fills it
104 /// (`ServerConfigOptions::inference_params`, threaded from four call
105 /// sites) is a larger removal than it looks and belongs in its own change.
106 /// Said out loud so the next reader does not wire something to it on the
107 /// assumption that it already does something.
108 pub inference_config: Option<InferenceConfig>,
109 /// Additional server-specific options (escape hatch).
110 pub extra_args: Vec<String>,
111 /// Directory for llama-server KV cache slot persistence (`--slot-save-path`).
112 ///
113 /// `None` means the disk slot-persistence feature is disabled — no
114 /// `--slot-save-path` flag is passed. Independent of [`Self::cache_ram_mb`]
115 /// / [`Self::cache_reuse`]: llama-server's own host-RAM prompt cache can be
116 /// tuned (or left at its built-in default) regardless of whether disk
117 /// persistence is on.
118 pub slot_save_path: Option<PathBuf>,
119 /// RAM budget in MiB for llama-server's own host-RAM prompt cache
120 /// (`--cache-ram`).
121 ///
122 /// `None` means no explicit flag is passed — llama-server's own built-in
123 /// default (8192 MiB) applies. `Some(n)` passes `--cache-ram n` directly;
124 /// `Some(0)` disables the cache.
125 pub cache_ram_mb: Option<u64>,
126 /// Minimum chunk size in tokens for KV-shift cache reuse past the first
127 /// prefix divergence point (`--cache-reuse`).
128 ///
129 /// `None` means no flag is passed (`--cache-reuse` off, llama-server
130 /// default `0`). `Some(n)` passes `--cache-reuse n`, letting llama-server
131 /// salvage matching KV chunks after an edited/summarized earlier message
132 /// instead of only reusing an unbroken prefix from token 0.
133 pub cache_reuse: Option<u32>,
134 /// K cache element type (`--cache-type-k`). `None` means no flag is
135 /// passed — llama-server's own `f16` default applies.
136 pub cache_type_k: Option<crate::cache_config::KvCacheType>,
137 /// V cache element type (`--cache-type-v`). Same semantics as
138 /// [`Self::cache_type_k`].
139 pub cache_type_v: Option<crate::cache_config::KvCacheType>,
140 /// Whether to lock the model in RAM (`--mlock`). Default: `false`.
141 pub mlock: bool,
142 /// Whether to serve this model in embedding mode (`--embeddings`).
143 ///
144 /// This is not an additive flag: llama-server reads it as *restrict to
145 /// only the embedding use case*, so a server started with it refuses
146 /// `/v1/chat/completions`, and one started without it answers
147 /// `/v1/embeddings` with a 501. Resolved from the model's `"embedding"`
148 /// tag, which makes the mode a property of which model is loaded rather
149 /// than of any individual request.
150 pub embeddings: bool,
151}
152
153impl ServerConfig {
154 /// Create a new server configuration with required fields.
155 #[must_use]
156 pub const fn new(
157 model_id: i64,
158 model_name: String,
159 model_path: PathBuf,
160 base_port: u16,
161 ) -> Self {
162 Self {
163 model_id,
164 model_name,
165 model_path,
166 port: None,
167 base_port,
168 context_size: None,
169 gpu_layers: None,
170 jinja: JinjaMode::Defer,
171 reasoning_format: None,
172 spec_draft_n_max: None,
173 spec_draft_p_min: None,
174 inference_config: None,
175 extra_args: Vec::new(),
176 slot_save_path: None,
177 cache_ram_mb: None,
178 cache_reuse: None,
179 cache_type_k: None,
180 cache_type_v: None,
181 mlock: false,
182 embeddings: false,
183 }
184 }
185
186 /// Set the port to listen on.
187 #[must_use]
188 pub const fn with_port(mut self, port: u16) -> Self {
189 self.port = Some(port);
190 self
191 }
192
193 /// Set the context size.
194 #[must_use]
195 pub const fn with_context_size(mut self, size: u64) -> Self {
196 self.context_size = Some(size);
197 self
198 }
199
200 /// Set the number of GPU layers.
201 #[must_use]
202 pub const fn with_gpu_layers(mut self, layers: i32) -> Self {
203 self.gpu_layers = Some(layers);
204 self
205 }
206
207 /// State this launch's position on Jinja templating.
208 ///
209 /// Takes the mode rather than defaulting to "on" because the caller that
210 /// has resolved it is the only one that knows which of the two falsy
211 /// answers it holds — see [`JinjaMode`].
212 #[must_use]
213 pub const fn with_jinja_mode(mut self, mode: JinjaMode) -> Self {
214 self.jinja = mode;
215 self
216 }
217
218 /// Serve this model in embedding mode (`--embeddings`).
219 ///
220 /// See [`Self::embeddings`] — this makes the server embeddings-only.
221 #[must_use]
222 pub const fn with_embeddings(mut self) -> Self {
223 self.embeddings = true;
224 self
225 }
226
227 /// Set the reasoning format (e.g., `"deepseek"`, `"none"`).
228 #[must_use]
229 pub fn with_reasoning_format(mut self, format: String) -> Self {
230 self.reasoning_format = Some(format);
231 self
232 }
233
234 /// Enable MTP speculative decoding with the given draft token count.
235 ///
236 /// This causes `--spec-type draft-mtp` and `--spec-draft-n-max n` to be
237 /// passed to llama-server. Call [`Self::with_spec_draft_p_min`] to also
238 /// set the acceptance probability threshold (defaults to 0.75).
239 #[must_use]
240 pub const fn with_spec_draft_n_max(mut self, n: u32) -> Self {
241 self.spec_draft_n_max = Some(n);
242 self
243 }
244
245 /// Set the minimum acceptance probability for MTP draft tokens.
246 ///
247 /// Has no effect unless `spec_draft_n_max` is also set. Recommended
248 /// value is `0.75`; lower values trade quality for speed.
249 #[must_use]
250 pub const fn with_spec_draft_p_min(mut self, p: f32) -> Self {
251 self.spec_draft_p_min = Some(p);
252 self
253 }
254
255 /// Set inference sampling parameters.
256 #[must_use]
257 pub const fn with_inference_config(mut self, config: InferenceConfig) -> Self {
258 self.inference_config = Some(config);
259 self
260 }
261
262 /// Set the KV cache slot-save directory (`--slot-save-path`).
263 ///
264 /// `None` disables the disk slot-persistence feature (no
265 /// `--slot-save-path` flag emitted). Independent of
266 /// [`Self::with_cache_ram_mb`] / [`Self::with_cache_reuse`].
267 #[must_use]
268 pub fn with_slot_save_path(mut self, path: Option<PathBuf>) -> Self {
269 self.slot_save_path = path;
270 self
271 }
272
273 /// Set the RAM budget (in MiB) for llama-server's own host-RAM prompt
274 /// cache (`--cache-ram`). `None` leaves llama-server's built-in default.
275 #[must_use]
276 pub const fn with_cache_ram_mb(mut self, mb: u64) -> Self {
277 self.cache_ram_mb = Some(mb);
278 self
279 }
280
281 /// Set the minimum chunk size (in tokens) for KV-shift cache reuse
282 /// (`--cache-reuse`). `None` leaves the feature off.
283 #[must_use]
284 pub const fn with_cache_reuse(mut self, n: u32) -> Self {
285 self.cache_reuse = Some(n);
286 self
287 }
288
289 /// Set the K cache element type (`--cache-type-k`).
290 #[must_use]
291 pub const fn with_cache_type_k(mut self, t: crate::cache_config::KvCacheType) -> Self {
292 self.cache_type_k = Some(t);
293 self
294 }
295
296 /// Set the V cache element type (`--cache-type-v`).
297 #[must_use]
298 pub const fn with_cache_type_v(mut self, t: crate::cache_config::KvCacheType) -> Self {
299 self.cache_type_v = Some(t);
300 self
301 }
302
303 /// Enable memory lock (`--mlock`).
304 #[must_use]
305 pub const fn with_mlock(mut self) -> Self {
306 self.mlock = true;
307 self
308 }
309}
310
311/// Handle to a running server process.
312///
313/// This is an opaque handle that implementations use to track processes.
314/// It contains enough information to identify and manage the process.
315#[derive(Debug, Clone, Serialize, Deserialize)]
316pub struct ProcessHandle {
317 /// Database ID of the model being served.
318 pub model_id: i64,
319 /// Human-readable model name.
320 pub model_name: String,
321 /// Process ID (if running on local system).
322 pub pid: Option<u32>,
323 /// Port the server is listening on.
324 pub port: u16,
325 /// Unix timestamp (seconds) when the server was started.
326 pub started_at: u64,
327}
328
329impl ProcessHandle {
330 /// Create a new process handle.
331 #[must_use]
332 pub const fn new(
333 model_id: i64,
334 model_name: String,
335 pid: Option<u32>,
336 port: u16,
337 started_at: u64,
338 ) -> Self {
339 Self {
340 model_id,
341 model_name,
342 pid,
343 port,
344 started_at,
345 }
346 }
347}