Merge 4845b90d9b5f1ef643355a4886d745c1fdd6b4fb into 254f6b986720c92ddf97fbb1a6a6465da8e87e29

This commit is contained in:
ゆり 2025-12-25 00:07:15 +00:00 committed by GitHub
commit c36d66b334
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -58,7 +58,9 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
def __reduce__(self):
return get_cached_tokenizer, (tokenizer,)
CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
# Preserve original class name for HuggingFace compatibility.
# Some processors validate tokenizer type by checking __name__.
CachedTokenizer.__name__ = tokenizer.__class__.__name__
cached_tokenizer.__class__ = CachedTokenizer
return cached_tokenizer