Add GLM 5.3 Flash support
This commit is contained in:
1 parent
9a33c61ea6
commit
46d6a976a5
31 files changed
+7806
-447
No files matched your search
+20
-1
@@ -1,5 +1,8 @@
|
||||
use super::gguf::Gguf;
|
||||
use super::{ChatTurn, ModelFamily};
|
||||
use super::{
|
||||
ChatTurn, ModelFamily, VISION_END_TOKEN, VISION_IMAGE_TOKEN, VISION_START_TOKEN,
|
||||
VISION_TOKEN_END, VISION_TOKEN_START,
|
||||
};
|
||||
use crate::settings::ReasoningMode;
|
||||
use std::collections::HashMap;
|
||||
|
||||
@@ -160,6 +163,22 @@ impl Tokenizer {
|
||||
let mut span = 0;
|
||||
let mut position = 0;
|
||||
while position < bytes.len() {
|
||||
if bytes[position..].starts_with(VISION_TOKEN_START.as_bytes())
|
||||
&& let Some(relative_end) =
|
||||
text[position + VISION_TOKEN_START.len()..].find(VISION_TOKEN_END)
|
||||
{
|
||||
let count_start = position + VISION_TOKEN_START.len();
|
||||
let count_end = count_start + relative_end;
|
||||
if let Ok(count) = text[count_start..count_end].parse::<usize>() {
|
||||
self.tokenize_plain(&text[span..position], &mut output);
|
||||
output.push(VISION_START_TOKEN);
|
||||
output.extend(std::iter::repeat_n(VISION_IMAGE_TOKEN, count));
|
||||
output.push(VISION_END_TOKEN);
|
||||
position = count_end + VISION_TOKEN_END.len();
|
||||
span = position;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let special = self
|
||||
.rendered_specials
|
||||
.iter()
|
||||
|
||||
Reference in new issue
Block a user