130 lines
4.7 KiB
Rust
130 lines
4.7 KiB
Rust
//! Token-exact cache compatibility across the Qwen split correction.
|
|
use super::*;
|
|
use crate::settings::{
|
|
GenerationPreferences, ReasoningMode, RuntimePreferences, SpeculativePreferences,
|
|
effective_settings,
|
|
};
|
|
use std::path::PathBuf;
|
|
|
|
#[test]
|
|
#[ignore = "installed model; serial external memory supervisor required"]
|
|
fn issue111_token_prefix_cache() {
|
|
let root = PathBuf::from(std::env::var_os("DS4_QWEN38_ARTIFACTS").unwrap());
|
|
let oracle = PathBuf::from(std::env::var_os("DS4_ISSUE111_ORACLE").unwrap());
|
|
let directory = PathBuf::from(std::env::var_os("DS4_ISSUE111_DIR").unwrap());
|
|
std::fs::create_dir_all(&directory).unwrap();
|
|
let requests: serde_json::Value =
|
|
serde_json::from_slice(&std::fs::read(oracle.join("requests.json")).unwrap()).unwrap();
|
|
let legacy: Vec<Vec<i32>> =
|
|
serde_json::from_slice(&std::fs::read(oracle.join("legacy-requests.json")).unwrap())
|
|
.unwrap();
|
|
let case = &requests[1];
|
|
let ids: Vec<i32> = serde_json::from_value(case["ids"].clone()).unwrap();
|
|
let settings = effective_settings(
|
|
crate::model::ModelChoice::Qwen38FlashNext,
|
|
&GenerationPreferences {
|
|
context_tokens: 8192,
|
|
max_generated_tokens: 2,
|
|
system_prompt: String::new(),
|
|
temperature: Some(0.),
|
|
top_p: Some(0.95),
|
|
min_p: Some(0.),
|
|
seed: Some(1),
|
|
reasoning_mode: ReasoningMode::Low,
|
|
},
|
|
&RuntimePreferences {
|
|
speculative: SpeculativePreferences {
|
|
glm_mtp: true,
|
|
..Default::default()
|
|
},
|
|
..Default::default()
|
|
},
|
|
root.parent().unwrap(),
|
|
)
|
|
.unwrap();
|
|
let messages = case["messages"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|m| ChatTurn {
|
|
user: m["role"] == "user",
|
|
tool: false,
|
|
system: false,
|
|
skip_previous_eos: false,
|
|
reasoning: m["reasoning_content"].as_str().map(str::to_owned),
|
|
reasoning_complete: true,
|
|
content: m["content"].as_str().unwrap().into(),
|
|
})
|
|
.collect::<Vec<_>>();
|
|
let mut owner = QwenExecutor::open(&settings.engine).unwrap();
|
|
owner.policy.store_min_suffix = Some(0);
|
|
assert_eq!(
|
|
owner
|
|
.metadata
|
|
.tokenizer
|
|
.encode_conversation("", &messages, ReasoningMode::Low),
|
|
ids
|
|
);
|
|
assert_ne!(legacy[1], ids);
|
|
owner.prefill(&legacy[1], |_| true).unwrap();
|
|
owner
|
|
.save_checkpoint(&directory.join("old-prefix.bin"), [111; 32], &mut |_| {})
|
|
.unwrap();
|
|
owner.reset().unwrap();
|
|
assert!(
|
|
owner
|
|
.load_checkpoint(&directory.join("old-prefix.bin"), &mut |_| {})
|
|
.unwrap()
|
|
);
|
|
assert_eq!(owner.tokens(), legacy[1]);
|
|
assert_eq!(owner.align_prompt(&ids).unwrap(), 0);
|
|
assert!(owner.live.current.is_none());
|
|
let metrics = Metrics::new(&directory.join("metrics"));
|
|
let run = |owner: &mut QwenExecutor, messages: &[ChatTurn]| {
|
|
let (out, complete) = owner
|
|
.generate(
|
|
messages,
|
|
&settings.turn,
|
|
&AtomicBool::new(false),
|
|
&metrics,
|
|
|_, _, _| {},
|
|
|_, _| {},
|
|
)
|
|
.unwrap();
|
|
assert!(complete);
|
|
out
|
|
};
|
|
let cold = run(&mut owner, &messages);
|
|
assert_eq!(cold.cached_tokens, 0);
|
|
assert_eq!(cold.prompt_tokens as usize, ids.len());
|
|
assert!(owner.tokens().starts_with(&ids));
|
|
// Preserve the completed-turn tag: a fabricated tag forces a raw-history
|
|
// rebuild and does not exercise compatible live-generation continuation.
|
|
let tag = owner.checkpoint_tag();
|
|
owner
|
|
.save_checkpoint(&directory.join("new-prefix.bin"), tag, &mut |_| {})
|
|
.unwrap();
|
|
let mut next = messages.clone();
|
|
next.push(cold.message.clone());
|
|
next.push(ChatTurn {
|
|
content: "Continue with Ⅷ and ².".into(),
|
|
..messages[0].clone()
|
|
});
|
|
let warm = run(&mut owner, &next);
|
|
assert!(warm.cached_tokens >= cold.prompt_tokens);
|
|
let expected = super::issue107_product::state(&owner);
|
|
owner.reset().unwrap();
|
|
assert!(
|
|
owner
|
|
.load_checkpoint(&directory.join("new-prefix.bin"), &mut |_| {})
|
|
.unwrap()
|
|
);
|
|
let reopened = run(&mut owner, &next);
|
|
assert_eq!(warm.cached_tokens, reopened.cached_tokens);
|
|
assert_eq!(expected, super::issue107_product::state(&owner));
|
|
println!(
|
|
"{}",
|
|
serde_json::json!({"case":"token_cache", "old_tokens":legacy[1].len(),"new_tokens":ids.len(),"old_prefix_miss":true,"warm_cached":warm.cached_tokens,"disk_cached":reopened.cached_tokens,"warm_disk_state_exact":true})
|
|
);
|
|
}
|