Match Qwen Unicode number token boundaries

This commit is contained in:
Georg Bauer committed 2026-10-04 15:37:04 +02:00
1 parent 990abb4755
commit 8965e77ea1
24 files changed
+27460 -3

No files matched your search

@@ -0,0 +1 @@
{"scalars": 1112064, "pieces": 2082513, "sha256": "7ef8755fb56062d9695b38bc132f325ea59c3c65f198e1b1ab20bff1afef632a", "pattern": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}