gglib_core/normalize/
residue.rs

1//! Chunk-safe scanner for dialect markup that reached client-visible text.
2//!
3//! Normalization is heuristic: detection can miss a dialect, a template can
4//! lie, a model can emit markup its spec does not describe.  When that
5//! happens the raw markup flows to the client as ordinary text — silently.
6//! [`ResidueScanner`] watches the *post-normalization* client-visible text
7//! for known tool-call markers so the proxy can turn that silent breakage
8//! into a logged, counted, dashboard-visible signal (the "dialect drift
9//! alarm").
10//!
11//! The scanner never alters the stream — it only observes.  A hit means
12//! "a human should look at this model's dialect handling", not "the proxy
13//! will fix it".
14//!
15//! ## Marker set
16//!
17//! The scan looks for the union of:
18//!
19//! - the active [`DialectSpec`]'s own markers, when the model has one —
20//!   markup surviving *its own* parser is the clearest drift signal; and
21//! - [`KNOWN_RESIDUE_MARKERS`], a curated list of tool-call markers from
22//!   dialects in the wild.  This deliberately duplicates a handful of
23//!   literals from `gglib-gguf`'s detection pattern tables: core cannot
24//!   depend on `gglib-gguf`, the list is small and changes rarely, and a
25//!   doc cross-reference keeps the two in sight of each other.
26//!
27//! ## Chunk safety
28//!
29//! Like the delimited parser, the scanner sees SSE-sized fragments: a
30//! marker may straddle chunk boundaries.  [`ResidueScanner::feed`] retains
31//! the longest tail of the previous chunk that could be a marker prefix
32//! (at most `max marker length − 1` bytes, cut on a char boundary) and
33//! prepends it to the next chunk, so a split marker is still found.  The
34//! first hit is sticky; scanning short-circuits afterwards.
35
36use crate::domain::dialect::DialectSpec;
37
38/// Tool-call markers from dialects observed in the wild, scanned in
39/// addition to the active spec's own markers.
40///
41/// Curated from `gglib-gguf`'s detection pattern tables
42/// (`crates/gglib-gguf/src/capabilities/patterns.rs`) — see the module
43/// docs for why this small duplication is deliberate.  Only *tool-call*
44/// markers belong here: reasoning tags are handled (and stripped)
45/// elsewhere, and generic XML would false-positive on ordinary prose.
46pub const KNOWN_RESIDUE_MARKERS: &[&str] = &[
47    "<tool_call>",
48    "</tool_call>",
49    "<function=",
50    "[TOOL_CALLS]",
51    "<｜tool▁calls▁begin｜>",
52    "<｜tool▁call▁begin｜>",
53    "<|python_tag|>",
54    "functools[",
55];
56
57/// Chunk-safe residue scanner.  See the module docs.
58#[derive(Debug)]
59pub struct ResidueScanner {
60    /// Markers to scan for: the active spec's (if any) ∪ the known set.
61    markers: Vec<String>,
62    /// Longest marker length in bytes — bounds the retained tail.
63    max_marker_len: usize,
64    /// Tail of the previous chunk that could still open a marker.
65    tail: String,
66    /// First marker found, if any.  Sticky.
67    hit: Option<String>,
68}
69
70impl ResidueScanner {
71    /// Build a scanner for a model with the given resolved dialect.
72    #[must_use]
73    pub fn new(dialect: Option<&DialectSpec>) -> Self {
74        let mut markers: Vec<String> = KNOWN_RESIDUE_MARKERS
75            .iter()
76            .map(|m| (*m).to_owned())
77            .collect();
78        if let Some(spec) = dialect {
79            for m in [&spec.tool_open, &spec.tool_close] {
80                if !m.is_empty() && !markers.contains(m) {
81                    markers.push(m.clone());
82                }
83            }
84        }
85        let max_marker_len = markers.iter().map(String::len).max().unwrap_or(0);
86        Self {
87            markers,
88            max_marker_len,
89            tail: String::new(),
90            hit: None,
91        }
92    }
93
94    /// Scan one chunk of client-visible text.
95    ///
96    /// Cheap after the first hit (immediately returns).  Never alters the
97    /// text — the caller forwards it regardless.
98    pub fn feed(&mut self, chunk: &str) {
99        if self.hit.is_some() || chunk.is_empty() {
100            return;
101        }
102
103        // Prepend the held-back tail so straddled markers are visible.
104        let window = if self.tail.is_empty() {
105            chunk.to_owned()
106        } else {
107            let mut w = std::mem::take(&mut self.tail);
108            w.push_str(chunk);
109            w
110        };
111
112        if let Some(found) = self.scan(&window) {
113            self.hit = Some(found);
114            self.tail.clear();
115            return;
116        }
117
118        // Retain the longest tail that could still be a marker prefix,
119        // cut on a char boundary.
120        let mut keep = window.len().min(self.max_marker_len.saturating_sub(1));
121        while keep > 0 && !window.is_char_boundary(window.len() - keep) {
122            keep -= 1;
123        }
124        window[window.len() - keep..].clone_into(&mut self.tail);
125    }
126
127    /// The first marker seen in the fed text, if any.
128    #[must_use]
129    pub fn hit(&self) -> Option<&str> {
130        self.hit.as_deref()
131    }
132
133    fn scan(&self, window: &str) -> Option<String> {
134        self.markers
135            .iter()
136            .find(|m| window.contains(m.as_str()))
137            .cloned()
138    }
139}
140
141/// One-shot scan of a complete (non-streaming) text for residue markers.
142///
143/// Same marker set as [`ResidueScanner`]; chunk safety is trivially
144/// satisfied because the whole text is one window.
145#[must_use]
146pub fn scan_complete(text: &str, dialect: Option<&DialectSpec>) -> Option<String> {
147    if text.is_empty() {
148        return None;
149    }
150    let scanner = ResidueScanner::new(dialect);
151    scanner.scan(text)
152}
153
154#[cfg(test)]
155mod tests {
156    use super::*;
157
158    fn feed_all(scanner: &mut ResidueScanner, chunks: &[&str]) {
159        for c in chunks {
160            scanner.feed(c);
161        }
162    }
163
164    #[test]
165    fn clean_text_never_hits() {
166        let mut s = ResidueScanner::new(None);
167        feed_all(&mut s, &["hello ", "world, no markup here"]);
168        assert_eq!(s.hit(), None);
169    }
170
171    #[test]
172    fn known_marker_in_one_chunk_hits() {
173        let mut s = ResidueScanner::new(None);
174        s.feed(r#"text <tool_call>{"name":"x"}</tool_call> more"#);
175        assert_eq!(s.hit(), Some("<tool_call>"));
176    }
177
178    #[test]
179    fn marker_straddling_two_chunks_hits() {
180        let mut s = ResidueScanner::new(None);
181        feed_all(&mut s, &["before <tool", "_call> after"]);
182        assert_eq!(s.hit(), Some("<tool_call>"));
183    }
184
185    #[test]
186    fn marker_straddling_three_chunks_hits() {
187        let mut s = ResidueScanner::new(None);
188        feed_all(&mut s, &["x<to", "ol_c", "all>y"]);
189        assert_eq!(s.hit(), Some("<tool_call>"));
190    }
191
192    #[test]
193    fn multibyte_marker_straddling_chunks_hits() {
194        // The deepseek markers are multibyte; split mid-scalar-boundary.
195        let mut s = ResidueScanner::new(None);
196        feed_all(&mut s, &["a<｜tool▁calls", "▁begin｜>b"]);
197        assert_eq!(s.hit(), Some("<｜tool▁calls▁begin｜>"));
198    }
199
200    #[test]
201    fn lookalike_prefix_that_never_completes_does_not_hit() {
202        let mut s = ResidueScanner::new(None);
203        feed_all(&mut s, &["before <tool", "s> after"]);
204        assert_eq!(s.hit(), None);
205    }
206
207    #[test]
208    fn spec_markers_are_included() {
209        let spec = DialectSpec {
210            tool_open: "«TC»".to_owned(),
211            tool_close: "«/TC»".to_owned(),
212            ..DialectSpec::qwen_xml()
213        };
214        let mut s = ResidueScanner::new(Some(&spec));
215        feed_all(&mut s, &["oops «T", "C» leaked"]);
216        assert_eq!(s.hit(), Some("«TC»"));
217    }
218
219    #[test]
220    fn hit_is_sticky() {
221        let mut s = ResidueScanner::new(None);
222        s.feed("<tool_call>");
223        s.feed("<function=");
224        assert_eq!(s.hit(), Some("<tool_call>"));
225    }
226
227    #[test]
228    fn scan_complete_matches_the_streaming_scanner() {
229        assert_eq!(
230            scan_complete("x <function=grep> y", None),
231            Some("<function=".to_owned())
232        );
233        assert_eq!(scan_complete("clean", None), None);
234        assert_eq!(scan_complete("", None), None);
235    }
236}
gglib_core/normalize/residue.rs

gglib_core/normalize/
residue.rs
