Remove all_special_tokens_extended from tokenizer code (#29686)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

tests/tokenization/test_cached_tokenizer.py

@@ -31,7 +31,6 @@ def _check_consistency(target: AnyTokenizer, expected: AnyTokenizer):
     # Cached attributes
     assert target.all_special_ids == expected.all_special_ids
     assert target.all_special_tokens == expected.all_special_tokens
-    assert target.all_special_tokens_extended == expected.all_special_tokens_extended
     assert target.get_vocab() == expected.get_vocab()
     assert len(target) == len(expected)
 

tests/tokenization/test_mistral_tokenizer.py

@@ -258,14 +258,8 @@ def mistral_tokenizer(request) -> MistralTokenizer:
 )
 class TestMistralTokenizer:
     def test_all_special_tokens(self, mistral_tokenizer: MistralTokenizer):
-        attributes = [
-            mistral_tokenizer.all_special_tokens,
-            mistral_tokenizer.all_special_tokens_extended,
-        ]
-
-        for attribute in attributes:
         if mistral_tokenizer.is_tekken:
-                assert attribute == [
+            assert mistral_tokenizer.all_special_tokens == [
                 "<unk>",
                 "<s>",
                 "</s>",
@@ -293,7 +287,7 @@ class TestMistralTokenizer:
                 "[/THINK]",
             ] + [f"<SPECIAL_{i}>" for i in range(36, 1000)]
         else:
-                assert attribute == [
+            assert mistral_tokenizer.all_special_tokens == [
                 "<s>",
                 "</s>",
                 "[INST]",

tests/tokenization/test_tokenizer_registry.py

@@ -15,10 +15,6 @@ class TestTokenizer(TokenizerBase):
     def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer":
         return TestTokenizer()
 
-    @property
-    def all_special_tokens_extended(self) -> list[str]:
-        raise NotImplementedError()
-
     @property
     def all_special_tokens(self) -> list[str]:
         raise NotImplementedError()

vllm/transformers_utils/tokenizer.py

@@ -96,7 +96,6 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
 
     tokenizer_all_special_ids = tokenizer.all_special_ids
     tokenizer_all_special_tokens = tokenizer.all_special_tokens
-    tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended
     tokenizer_vocab = tokenizer.get_vocab()
     tokenizer_len = len(tokenizer)
 
@@ -118,10 +117,6 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
         def all_special_tokens(self) -> list[str]:
             return tokenizer_all_special_tokens
 
-        @property
-        def all_special_tokens_extended(self) -> list[str]:
-            return tokenizer_all_special_tokens_extended
-
         @property
         def max_token_id(self) -> int:
             return max_token_id

vllm/transformers_utils/tokenizer_base.py

@@ -10,11 +10,6 @@ if TYPE_CHECKING:
 
 
 class TokenizerBase(ABC):
-    @property
-    @abstractmethod
-    def all_special_tokens_extended(self) -> list[str]:
-        raise NotImplementedError()
-
     @property
     @abstractmethod
     def all_special_tokens(self) -> list[str]:

vllm/transformers_utils/tokenizers/mistral.py

@@ -254,10 +254,6 @@ class MistralTokenizer(TokenizerBase):
 
     # the following attributes are set to fit vLLM's design and are used
     # by the structured output backends.
-    @property
-    def all_special_tokens_extended(self) -> list[str]:
-        return self.all_special_tokens
-
     @property
     def all_special_tokens(self) -> list[str]:
         return self._special_tokens