Support hydit checkpoint save.

This commit is contained in:
comfyanonymous 2024-07-25 11:03:51 -04:00
parent 18e958d6cb
commit 84d0062d68
2 changed files with 9 additions and 1 deletions

View File

@ -32,7 +32,7 @@ class RobertaWrapper(torch.nn.Module):
config = BertConfig(**config_dict) config = BertConfig(**config_dict)
with use_comfy_ops(operations, device, dtype): with use_comfy_ops(operations, device, dtype):
with modeling_utils.no_init_weights(): with modeling_utils.no_init_weights():
self.bert = BertModel(config) self.bert = BertModel(config, add_pooling_layer=False)
self.num_layers = config.num_hidden_layers self.num_layers = config.num_hidden_layers
@ -69,6 +69,8 @@ class MT5XLTokenizer(sd1_clip.SDTokenizer):
tokenizer = tokenizer_data.get("spiece_model", None) tokenizer = tokenizer_data.get("spiece_model", None)
super().__init__(tokenizer, pad_with_end=False, embedding_size=2048, embedding_key='mt5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256) super().__init__(tokenizer, pad_with_end=False, embedding_size=2048, embedding_key='mt5xl', tokenizer_class=SPieceTokenizer, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=256)
def state_dict(self):
return {"spiece_model": self.tokenizer.serialize_model()}
class HyditTokenizer: class HyditTokenizer:
def __init__(self, embedding_directory=None, tokenizer_data={}): def __init__(self, embedding_directory=None, tokenizer_data={}):
@ -85,6 +87,9 @@ class HyditTokenizer:
def untokenize(self, token_weight_pair): def untokenize(self, token_weight_pair):
return self.hydit_clip.untokenize(token_weight_pair) return self.hydit_clip.untokenize(token_weight_pair)
def state_dict(self):
return {"mt5xl.spiece_model": self.mt5xl.state_dict()["spiece_model"]}
class HyditModel(torch.nn.Module): class HyditModel(torch.nn.Module):
def __init__(self, device="cpu", dtype=None): def __init__(self, device="cpu", dtype=None):
super().__init__() super().__init__()

View File

@ -27,3 +27,6 @@ class SPieceTokenizer:
def __call__(self, string): def __call__(self, string):
out = self.tokenizer.encode(string) out = self.tokenizer.encode(string)
return {"input_ids": out} return {"input_ids": out}
def serialize_model(self):
return torch.ByteTensor(list(self.tokenizer.serialized_model_proto()))