Processors: don't default padding side (#33942)

* don't default padding side

* fix
This commit is contained in:
Raushan Turganbay 2024-10-08 10:58:49 +02:00 committed by GitHub
parent a3add29097
commit 0dbc7090ba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 6 additions and 7 deletions

View File

@ -235,9 +235,6 @@ class Idefics3Processor(ProcessorMixin):
**kwargs,
)
# Temporary fix for "padding_side" in init_kwargs
output_kwargs["text_kwargs"].pop("padding_side", None)
image_seq_len = image_seq_len if image_seq_len is not None else self.image_seq_len
n_images_in_text = []

View File

@ -172,8 +172,6 @@ class LlavaOnevisionProcessor(ProcessorMixin):
num_video_tokens = (num_frames * pooled_height_width * pooled_height_width) + 1 # +1 for newline token
text = [sample.replace(self.video_token, self.video_token * num_video_tokens) for sample in text]
# Padding side can be in TextKwargs but is not accepted by the tokenizer
_ = output_kwargs["text_kwargs"].pop("padding_side", None)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
return BatchFeature(data={**text_inputs, **image_inputs, **video_inputs})

View File

@ -150,7 +150,6 @@ class Qwen2VLProcessor(ProcessorMixin):
index += 1
text[i] = text[i].replace("<|placeholder|>", "<|video_pad|>")
_ = output_kwargs["text_kwargs"].pop("padding_side", None)
text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs})

View File

@ -829,7 +829,12 @@ class ProcessorMixin(PushToHubMixin):
for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys():
# init with tokenizer init kwargs if necessary
if modality_key in tokenizer_init_kwargs:
default_kwargs[modality][modality_key] = tokenizer_init_kwargs[modality_key]
value = (
getattr(self.tokenizer, modality_key)
if hasattr(self.tokenizer, modality_key)
else tokenizer_init_kwargs[modality_key]
)
default_kwargs[modality][modality_key] = value
# now defaults kwargs are updated with the tokenizers defaults.
# pass defaults to output dictionary
output_kwargs.update(default_kwargs)