riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-04 13:20:12 +06:00
Code Issues Actions 21 Packages Projects Releases Wiki Activity

Examples: add Bloom support for token classification (#18632)

Browse Source 
* examples: add Bloom support for token classification (FLAX, PyTorch and TensorFlow)

* examples: remove support for Bloom in token classication (FLAX and TensorFlow currently have no support for it)
This commit is contained in:
Stefan Schweter 2022-08-17 09:50:57 +02:00 committed by GitHub
parent 6d175c1129
commit 358478e729
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/pytorch/token-classification/run_ner.py
View File

@ -348,7 +348,7 @@ def main():
) )
tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path tokenizer_name_or_path = model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path
if config.model_type in {"gpt2", "roberta"}: if config.model_type in {"bloom", "gpt2", "roberta"}:
tokenizer = AutoTokenizer.from_pretrained( tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name_or_path, tokenizer_name_or_path,
cache_dir=model_args.cache_dir, cache_dir=model_args.cache_dir,

2
examples/pytorch/token-classification/run_ner_no_trainer.py
View File

@ -398,7 +398,7 @@ def main():
"You can do it from another script, save it, and load it from here, using --tokenizer_name." "You can do it from another script, save it, and load it from here, using --tokenizer_name."
) )
if config.model_type in {"gpt2", "roberta"}: if config.model_type in {"bloom", "gpt2", "roberta"}:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True) tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True)
else: else:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True) tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True)