Select thinking mode from the chat
This commit is contained in:
@@ -8,6 +8,7 @@ pub(crate) const REASONING_MODES: [ReasoningMode; 3] = [
|
||||
ReasoningMode::Max,
|
||||
ReasoningMode::Direct,
|
||||
];
|
||||
const STANDARD_REASONING_MODES: [ReasoningMode; 2] = [ReasoningMode::High, ReasoningMode::Direct];
|
||||
const THINK_MAX_MIN_CONTEXT: i32 = 393_216;
|
||||
const MAX_CPU_THREADS: u32 = 32;
|
||||
const MAX_MTP_DRAFT_TOKENS: i32 = 16;
|
||||
@@ -555,6 +556,23 @@ impl Default for GenerationPreferences {
|
||||
}
|
||||
|
||||
impl GenerationPreferences {
|
||||
pub(crate) fn effective_reasoning_mode(&self) -> ReasoningMode {
|
||||
if self.reasoning_mode == ReasoningMode::Max && self.context_tokens < THINK_MAX_MIN_CONTEXT
|
||||
{
|
||||
ReasoningMode::High
|
||||
} else {
|
||||
self.reasoning_mode
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn supported_reasoning_modes(&self) -> &'static [ReasoningMode] {
|
||||
if self.context_tokens >= THINK_MAX_MIN_CONTEXT {
|
||||
&REASONING_MODES
|
||||
} else {
|
||||
&STANDARD_REASONING_MODES
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn validate(&self) -> Result<(), String> {
|
||||
if self.context_tokens <= 0 {
|
||||
return Err("Context tokens must be a positive whole number.".into());
|
||||
@@ -588,13 +606,7 @@ impl GenerationPreferences {
|
||||
top_k: 0,
|
||||
stops: Vec::new(),
|
||||
seed: self.seed,
|
||||
reasoning_mode: if self.reasoning_mode == ReasoningMode::Max
|
||||
&& self.context_tokens < THINK_MAX_MIN_CONTEXT
|
||||
{
|
||||
ReasoningMode::High
|
||||
} else {
|
||||
self.reasoning_mode
|
||||
},
|
||||
reasoning_mode: self.effective_reasoning_mode(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -680,6 +692,11 @@ mod tests {
|
||||
|
||||
let glm = defaults.turn_settings(ModelChoice::Glm52, cache);
|
||||
assert_eq!((glm.temperature, glm.top_p, glm.min_p), (1.0, 0.95, 0.0));
|
||||
assert_eq!(
|
||||
defaults.supported_reasoning_modes(),
|
||||
&[ReasoningMode::High, ReasoningMode::Direct]
|
||||
);
|
||||
assert_eq!(defaults.effective_reasoning_mode(), ReasoningMode::High);
|
||||
|
||||
let explicit = GenerationPreferences {
|
||||
top_p: Some(0.4),
|
||||
@@ -690,6 +707,38 @@ mod tests {
|
||||
let effective = explicit.turn_settings(ModelChoice::Glm52, cache);
|
||||
assert_eq!((effective.top_p, effective.min_p), (0.4, 0.2));
|
||||
assert_eq!(effective.reasoning_mode, ReasoningMode::High);
|
||||
|
||||
let long_context = GenerationPreferences {
|
||||
context_tokens: THINK_MAX_MIN_CONTEXT,
|
||||
reasoning_mode: ReasoningMode::Max,
|
||||
..GenerationPreferences::default()
|
||||
};
|
||||
assert_eq!(long_context.supported_reasoning_modes(), &REASONING_MODES);
|
||||
assert_eq!(long_context.effective_reasoning_mode(), ReasoningMode::Max);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_selection_reaches_every_model_turn() {
|
||||
let cache = KvCachePreferences::default().settings();
|
||||
for model in crate::model::MODEL_CHOICES {
|
||||
for mode in [ReasoningMode::Direct, ReasoningMode::High] {
|
||||
let generation = GenerationPreferences {
|
||||
reasoning_mode: mode,
|
||||
..GenerationPreferences::default()
|
||||
};
|
||||
assert_eq!(generation.turn_settings(model, cache).reasoning_mode, mode);
|
||||
}
|
||||
|
||||
let generation = GenerationPreferences {
|
||||
context_tokens: THINK_MAX_MIN_CONTEXT,
|
||||
reasoning_mode: ReasoningMode::Max,
|
||||
..GenerationPreferences::default()
|
||||
};
|
||||
assert_eq!(
|
||||
generation.turn_settings(model, cache).reasoning_mode,
|
||||
ReasoningMode::Max
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user