xinyun/ComfyUI
1
0
Fork 0
mirror of https://git.datalinker.icu/comfyanonymous/ComfyUI synced 2025-12-11 06:54:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update qwen tokenizer to add qwen 3 tokens. (#11029)

Browse Source 
Doesn't actually change anything for current workflows because none of the
current models have a template with the think tokens.
This commit is contained in:
comfyanonymous 2025-12-01 14:13:48 -08:00 committed by GitHub
parent 7dbd5dfe91
commit 2640acb31c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 8 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
comfy/text_encoders/qwen25_tokenizer/tokenizer_config.json
View File

@ -179,36 +179,36 @@
"special": false "special": false
}, },
"151665": { "151665": {
"content": "<|img|>", "content": "<tool_response>",
"lstrip": false, "lstrip": false,
"normalized": false, "normalized": false,
"rstrip": false, "rstrip": false,
"single_word": false, "single_word": false,
"special": true "special": false
}, },
"151666": { "151666": {
"content": "<|endofimg|>", "content": "</tool_response>",
"lstrip": false, "lstrip": false,
"normalized": false, "normalized": false,
"rstrip": false, "rstrip": false,
"single_word": false, "single_word": false,
"special": true "special": false
}, },
"151667": { "151667": {
"content": "<|meta|>", "content": "<think>",
"lstrip": false, "lstrip": false,
"normalized": false, "normalized": false,
"rstrip": false, "rstrip": false,
"single_word": false, "single_word": false,
"special": true "special": false
}, },
"151668": { "151668": {
"content": "<|endofmeta|>", "content": "</think>",
"lstrip": false, "lstrip": false,
"normalized": false, "normalized": false,
"rstrip": false, "rstrip": false,
"single_word": false, "single_word": false,
"special": true "special": false
} }
}, },
"additional_special_tokens": [ "additional_special_tokens": [