diff --git a/src/transformers/integrations/tiktoken.py b/src/transformers/integrations/tiktoken.py index 60f73392840..a5b40acfbae 100644 --- a/src/transformers/integrations/tiktoken.py +++ b/src/transformers/integrations/tiktoken.py @@ -41,5 +41,5 @@ def convert_tiktoken_to_fast(encoding: Any, output_dir: str): tokenizer = TikTokenConverter( vocab_file=save_file_absolute, pattern=encoding._pat_str, additional_special_tokens=encoding._special_tokens - ).tokenizer() + ).converted() tokenizer.save(output_file_absolute)