[Bugfix] Fixed mistral tokenizer path when pointing to file (#17457)

Signed-off-by: Pete Savage <psavage@redhat.com>
This commit is contained in:
Pete Savage 2025-04-30 16:08:37 +01:00 committed by GitHub
parent da4e7687b5
commit 739e03b344
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

View File

@ -227,6 +227,7 @@ class MistralTokenizer(TokenizerBase):
else:
assert Path(
path_or_repo_id).is_file(), f"Invalid path: {path_or_repo_id}"
tokenizer_file = str(Path(path_or_repo_id))
from mistral_common.tokens.tokenizers.mistral import (
MistralTokenizer as PublicMistralTokenizer)