Match Qwen Unicode number token boundaries

This commit is contained in:
Georg Bauer committed 2026-10-04 15:37:04 +02:00
1 parent 990abb4755
commit 8965e77ea1
24 files changed
+27460 -3

No files matched your search

@@ -0,0 +1,38 @@
[
{
"text": "́a á ́ १́b",
"ids": [
52033,
64,
264,
52033,
220,
52033,
220,
12256,
100,
52033,
65
],
"full_ids": [
52033,
64,
18135,
220,
52033,
220,
12256,
100,
52033,
65
],
"pieces": [
"́a",
" á",
" ́",
" ",
"१",
"́b"
]
}
]