riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

quality

Browse Source
This commit is contained in:
Julien Denize 2025-06-19 11:26:45 +02:00
parent 4679fe3bc1
commit d8215d3d74
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/transformers/tokenization_mistral_common.py
View File

@ -39,6 +39,7 @@ class MistralCommonTokenizer(PushToHubMixin):
Handles several methods for tokenization as well as methods for Handles several methods for tokenization as well as methods for
downloading/caching/loading pretrained tokenizers. downloading/caching/loading pretrained tokenizers.
""" """
def __init__( def __init__(
self, tokenizer_path: Union[str, os.PathLike, Path], mode: ValidationMode = ValidationMode.test, **kwargs self, tokenizer_path: Union[str, os.PathLike, Path], mode: ValidationMode = ValidationMode.test, **kwargs
): ):