//! Pure-Rust BERT uncased WordPiece tokenizer. //! //! Mirrors the Hugging Face `tokenizers` pipeline used by `BertTokenizerFast` //! with `do_lower_case=True`, step for step: //! //! 1. `BertNormalizer`: drop NUL/U+FFFD/control chars, map whitespace to ' ', //! pad CJK ideographs with spaces, NFD + drop non-spacing marks, lowercase //! (accent stripping happens *before* lowercasing, as in `tokenizers`). //! 2. `BertPreTokenizer`: split on whitespace, isolate punctuation. //! 3. `WordPiece`: greedy longest-match with `##` continuation, words longer //! than 100 chars become `[UNK]`. //! 4. `[CLS] … [SEP]` template with truncation to `max_len`. //! //! Parity is checked against fixtures generated by `tools/gen_fixtures.py`. use std::collections::HashMap; use std::fmt; use unicode_general_category::{get_general_category, GeneralCategory as Gc}; use unicode_normalization::UnicodeNormalization; const MAX_INPUT_CHARS_PER_WORD: usize = 100; #[derive(Debug, PartialEq)] pub enum VocabError { Empty, MissingSpecialToken(&'static str), Duplicate(String), } impl fmt::Display for VocabError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Empty => write!(f, "vocabulary is empty"), Self::MissingSpecialToken(t) => write!(f, "vocabulary is missing {t}"), Self::Duplicate(t) => write!(f, "vocabulary contains duplicate token {t:?}"), } } } impl std::error::Error for VocabError {} pub struct WordPieceTokenizer { vocab: HashMap, pad_id: u32, unk_id: u32, cls_id: u32, sep_id: u32, } /// Token ids of a single encoded text, without padding. pub type Encoding = Vec; impl WordPieceTokenizer { /// Build from the contents of a BERT `vocab.txt` (one token per line, /// id = line number). pub fn from_vocab(vocab_txt: &str) -> Result { let mut vocab = HashMap::with_capacity(32_000); for (id, line) in vocab_txt.lines().enumerate() { let token = line.trim_end_matches('\r'); if vocab.insert(token.to_string(), id as u32).is_some() { return Err(VocabError::Duplicate(token.to_string())); } } if vocab.is_empty() { return Err(VocabError::Empty); } let special = |t: &'static str| { vocab .get(t) .copied() .ok_or(VocabError::MissingSpecialToken(t)) }; Ok(Self { pad_id: special("[PAD]")?, unk_id: special("[UNK]")?, cls_id: special("[CLS]")?, sep_id: special("[SEP]")?, vocab, }) } pub fn pad_id(&self) -> u32 { self.pad_id } pub fn vocab_size(&self) -> usize { self.vocab.len() } /// Encode one text as `[CLS] tokens… [SEP]`, truncated to `max_len` ids. pub fn encode(&self, text: &str, max_len: usize) -> Encoding { let budget = max_len.saturating_sub(2); let mut ids = Vec::with_capacity(budget.min(64) + 2); ids.push(self.cls_id); 'words: for word in pre_tokenize(&normalize(text)) { for id in self.wordpiece(&word) { // ids[0] is [CLS]; stop once `budget` content tokens are in. if ids.len() > budget { break 'words; } ids.push(id); } } ids.push(self.sep_id); ids } fn wordpiece(&self, word: &str) -> Vec { let chars: Vec = word.chars().collect(); if chars.len() > MAX_INPUT_CHARS_PER_WORD { return vec![self.unk_id]; } let mut out = Vec::new(); let mut start = 0; let mut piece = String::with_capacity(word.len() + 2); while start < chars.len() { let mut end = chars.len(); let mut found = None; while start < end { piece.clear(); if start > 0 { piece.push_str("##"); } piece.extend(&chars[start..end]); if let Some(&id) = self.vocab.get(piece.as_str()) { found = Some(id); break; } end -= 1; } match found { Some(id) => { out.push(id); start = end; } None => return vec![self.unk_id], } } out } } fn is_bert_whitespace(c: char) -> bool { matches!(c, '\t' | '\n' | '\r') || c.is_whitespace() } fn is_bert_control(c: char) -> bool { if matches!(c, '\t' | '\n' | '\r') { return false; } matches!( get_general_category(c), Gc::Control | Gc::Format | Gc::Unassigned | Gc::PrivateUse | Gc::Surrogate ) } fn is_bert_punctuation(c: char) -> bool { c.is_ascii_punctuation() || matches!( get_general_category(c), Gc::ConnectorPunctuation | Gc::DashPunctuation | Gc::ClosePunctuation | Gc::FinalPunctuation | Gc::InitialPunctuation | Gc::OtherPunctuation | Gc::OpenPunctuation ) } fn is_cjk(c: char) -> bool { matches!( c as u32, 0x4E00..=0x9FFF | 0x3400..=0x4DBF | 0x20000..=0x2A6DF | 0x2A700..=0x2B73F | 0x2B740..=0x2B81F | 0x2B920..=0x2CEAF | 0xF900..=0xFAFF | 0x2F800..=0x2FA1F ) } /// BertNormalizer(clean_text, handle_chinese_chars, strip_accents, lowercase). fn normalize(text: &str) -> String { let mut cleaned = String::with_capacity(text.len()); for c in text.chars() { if c == '\0' || c == '\u{FFFD}' || is_bert_control(c) { continue; } if is_bert_whitespace(c) { cleaned.push(' '); } else if is_cjk(c) { cleaned.push(' '); cleaned.push(c); cleaned.push(' '); } else { cleaned.push(c); } } cleaned .nfd() .filter(|&c| get_general_category(c) != Gc::NonspacingMark) .flat_map(char::to_lowercase) .collect() } /// BertPreTokenizer: whitespace split + isolated punctuation. fn pre_tokenize(text: &str) -> Vec { let mut words = Vec::new(); let mut current = String::new(); for c in text.chars() { if c.is_whitespace() { if !current.is_empty() { words.push(std::mem::take(&mut current)); } } else if is_bert_punctuation(c) { if !current.is_empty() { words.push(std::mem::take(&mut current)); } words.push(c.to_string()); } else { current.push(c); } } if !current.is_empty() { words.push(current); } words } #[cfg(test)] mod tests { use super::*; const VOCAB: &str = include_str!("../model/vocab.txt"); fn tok() -> WordPieceTokenizer { WordPieceTokenizer::from_vocab(VOCAB).unwrap() } fn pieces(t: &WordPieceTokenizer, text: &str) -> Vec { let inv: HashMap = t.vocab.iter().map(|(k, &v)| (v, k.as_str())).collect(); t.encode(text, 128) .iter() .map(|id| inv[id].to_string()) .collect() } #[test] fn special_tokens_resolve_to_standard_ids() { let t = tok(); assert_eq!((t.pad_id, t.unk_id, t.cls_id, t.sep_id), (0, 100, 101, 102)); assert_eq!(t.vocab_size(), 30522); } #[test] fn empty_and_whitespace_input_is_cls_sep() { let t = tok(); assert_eq!(t.encode("", 128), vec![101, 102]); assert_eq!(t.encode(" \t\n ", 128), vec![101, 102]); } #[test] fn lowercases_and_splits_punctuation() { assert_eq!( pieces(&tok(), "Hello, World!"), ["[CLS]", "hello", ",", "world", "!", "[SEP]"] ); } #[test] fn strips_precomposed_accents() { // U+00E9 is precomposed; it must be decomposed (NFD) before stripping. assert_eq!(pieces(&tok(), "Café résumé"), pieces(&tok(), "cafe resume")); } #[test] fn turkish_dotted_capital_i_becomes_plain_i() { let p = pieces(&tok(), "İstanbul"); assert_eq!(p, pieces(&tok(), "istanbul")); } #[test] fn splits_wordpieces_with_continuation_prefix() { let p = pieces(&tok(), "unaffable"); assert!(p.len() > 3, "{p:?}"); assert!( p[2..p.len() - 1].iter().all(|s| s.starts_with("##")), "{p:?}" ); } #[test] fn cjk_characters_are_isolated() { let t = tok(); assert_eq!(t.encode("中文", 128).len(), 4); } #[test] fn unicode_punctuation_is_isolated() { assert_eq!(pieces(&tok(), "a—b"), ["[CLS]", "a", "—", "b", "[SEP]"]); } #[test] fn control_characters_are_removed() { assert_eq!(pieces(&tok(), "ab\u{200B}\u{0007}c"), pieces(&tok(), "abc")); } #[test] fn overlong_word_is_unk() { let t = tok(); assert_eq!(t.encode(&"a".repeat(101), 128), vec![101, 100, 102]); assert_ne!(t.encode(&"a".repeat(100), 128)[1], 100); } #[test] fn truncates_to_max_len_keeping_sep() { let t = tok(); let ids = t.encode(&"word ".repeat(500), 16); assert_eq!(ids.len(), 16); assert_eq!(ids[0], 101); assert_eq!(ids[15], 102); } #[test] fn rejects_vocab_without_special_tokens() { assert_eq!( WordPieceTokenizer::from_vocab("hello\nworld").err(), Some(VocabError::MissingSpecialToken("[PAD]")) ); assert_eq!( WordPieceTokenizer::from_vocab("").err(), Some(VocabError::Empty) ); } }