gglib_core/normalize/
registry.rs

1//! Dialect-spec parser dispatch.
2//!
3//! This module is the **single source of truth** for dialect selection, in
4//! two layers:
5//!
6//! * [`dialect_for_tags`] maps legacy `format:*` tags to a built-in
7//!   [`DialectSpec`] — the back-compat path for catalog rows persisted
8//!   before specs existed (and for models whose spec could not be derived).
9//! * [`get_parser`] turns a resolved spec into a parser.  Any spec drives
10//!   the delimited parser; no spec means the identity passthrough.
11//!
12//! Adding a new *builtin* dialect is one entry in [`dialect_for_tags`] (plus
13//! its constant in [`super::tags`]).  Template-derived dialects need no code
14//! at all: detection persists a spec and it arrives here as `Some`.
15//!
16//! No other crate looks at `format:*` tags for parser selection — they
17//! resolve a spec (usually via `ModelContext`) and call `get_parser`.  This
18//! keeps the dialect surface area tightly contained and prevents drift
19//! between callers.
20
21use super::parser::ToolCallParser;
22use super::parsers::{delimited::DelimitedToolCallParser, standard::StandardJsonParser};
23use super::tags;
24use crate::domain::dialect::DialectSpec;
25
26/// Map legacy `format:*` tags to a built-in [`DialectSpec`].
27///
28/// Tags are scanned in the listed order and the first recognised tag wins.
29/// Returns `None` for models with no recognised tag — the common case.
30///
31/// Both [`tags::FORMAT_QWEN_XML`] and [`tags::FORMAT_HERMES`] map to the
32/// built-in Qwen spec: the envelope-plus-JSON dialect is shared, and the
33/// spec's inner-XML fallback codec is Hermes-style to begin with.
34#[must_use]
35pub fn dialect_for_tags(model_tags: &[String]) -> Option<DialectSpec> {
36    for t in model_tags {
37        // Future builtin dialects slot in here, one arm each.
38        match t.as_str() {
39            tags::FORMAT_QWEN_XML | tags::FORMAT_HERMES => return Some(DialectSpec::qwen_xml()),
40            _ => {}
41        }
42    }
43    None
44}
45
46/// Pick a parser for a resolved dialect.
47///
48/// `Some(spec)` — from the model's persisted spec or the
49/// [`dialect_for_tags`] fallback — yields a
50/// [`DelimitedToolCallParser`] configured with it; `None` yields the
51/// identity-passthrough [`StandardJsonParser`].
52///
53/// The returned trait object is `Send` because [`ToolCallParser`] requires
54/// `Send`; this lets `NormalizingStream` live on a tokio task without
55/// adding a separate bound.
56#[must_use]
57pub fn get_parser(dialect: Option<&DialectSpec>) -> Box<dyn ToolCallParser> {
58    match dialect {
59        Some(spec) => Box::new(DelimitedToolCallParser::new(spec.clone())),
60        None => Box::new(StandardJsonParser::new()),
61    }
62}
63
64#[cfg(test)]
65mod tests {
66    use super::*;
67
68    #[test]
69    fn no_dialect_yields_standard_parser() {
70        let mut p = get_parser(None);
71        let out = p.push_text("hello");
72        assert_eq!(out.forward_text, "hello");
73    }
74
75    #[test]
76    fn qwen_tag_maps_to_the_builtin_spec_and_parses() {
77        let dialect = dialect_for_tags(&[tags::FORMAT_QWEN_XML.to_owned()]);
78        assert_eq!(dialect, Some(DialectSpec::qwen_xml()));
79
80        let mut p = get_parser(dialect.as_ref());
81        let out = p.push_text(r#"<tool_call>{"name":"x","arguments":{}}</tool_call>"#);
82        let f = p.finish();
83        assert_eq!(out.tool_calls.len(), 1);
84        assert!(
85            f.tool_calls.is_empty(),
86            "tool calls flush in push, not finish"
87        );
88    }
89
90    /// The hermes tag has been emitted by detection since it existed, but
91    /// nothing consumed it — models carrying it leaked raw `<tool_call>`
92    /// markup. It now maps to the same builtin spec as the qwen tag.
93    #[test]
94    fn hermes_tag_maps_to_the_builtin_spec() {
95        let dialect = dialect_for_tags(&[tags::FORMAT_HERMES.to_owned()]);
96        assert_eq!(dialect, Some(DialectSpec::qwen_xml()));
97    }
98
99    #[test]
100    fn unknown_tags_yield_no_dialect() {
101        assert_eq!(
102            dialect_for_tags(&["format:does-not-exist".to_owned()]),
103            None
104        );
105        assert_eq!(dialect_for_tags(&[]), None);
106
107        let mut p = get_parser(None);
108        let out = p.push_text("<tool_call>passthrough</tool_call>");
109        assert_eq!(out.forward_text, "<tool_call>passthrough</tool_call>");
110    }
111
112    #[test]
113    fn first_recognised_tag_wins() {
114        let tags_v = vec![
115            "format:does-not-exist".to_owned(),
116            tags::FORMAT_QWEN_XML.to_owned(),
117        ];
118        let dialect = dialect_for_tags(&tags_v);
119        let mut p = get_parser(dialect.as_ref());
120        let out = p.push_text(r#"<tool_call>{"name":"x","arguments":{}}</tool_call>"#);
121        assert_eq!(out.forward_text, "");
122        assert_eq!(out.tool_calls.len(), 1);
123    }
124
125    /// An explicit spec — the template-derived path — needs no tag at all.
126    #[test]
127    fn an_explicit_spec_drives_the_delimited_parser() {
128        let spec = DialectSpec {
129            tool_open: "«TC»".to_owned(),
130            tool_close: "«/TC»".to_owned(),
131            ..DialectSpec::qwen_xml()
132        };
133        let mut p = get_parser(Some(&spec));
134        let out = p.push_text(r#"«TC»{"name":"x","arguments":{}}«/TC»"#);
135        assert_eq!(out.tool_calls.len(), 1);
136        assert_eq!(out.forward_text, "");
137    }
138}
gglib_core/normalize/registry.rs

gglib_core/normalize/
registry.rs
