riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-20 04:58:22 +06:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

Remove unused code (#36459)

Browse Source
This commit is contained in:
Matt 2025-03-03 18:31:10 +00:00 committed by GitHub
parent 9fe82793ee
commit 6aa9888463
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 0 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/transformers/tokenization_utils_base.py
View File

@ -2012,7 +2012,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
# Get files from url, cache, or disk depending on the case # Get files from url, cache, or disk depending on the case
resolved_vocab_files = {} resolved_vocab_files = {}
unresolved_files = []
for file_id, file_path in vocab_files.items(): for file_id, file_path in vocab_files.items():
if file_path is None: if file_path is None:
resolved_vocab_files[file_id] = None resolved_vocab_files[file_id] = None
@ -2041,12 +2040,6 @@ class PreTrainedTokenizerBase(SpecialTokensMixin, PushToHubMixin):
) )
commit_hash = extract_commit_hash(resolved_vocab_files[file_id], commit_hash) commit_hash = extract_commit_hash(resolved_vocab_files[file_id], commit_hash)
if len(unresolved_files) > 0:
logger.info(
f"Can't load following files from cache: {unresolved_files} and cannot check if these "
"files are necessary for the tokenizer to operate."
)
# If one passes a GGUF file path to `gguf_file` there is no need for this check as the tokenizer will be # If one passes a GGUF file path to `gguf_file` there is no need for this check as the tokenizer will be
# loaded directly from the GGUF file. # loaded directly from the GGUF file.
if all(full_file_name is None for full_file_name in resolved_vocab_files.values()) and not gguf_file: if all(full_file_name is None for full_file_name in resolved_vocab_files.values()) and not gguf_file: