mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-29 17:22:25 +06:00
fa84ae26d6
This is the result of:
$ black --line-length 119 examples templates transformers utils hubconf.py setup.py
There's a lot of fairly long lines in the project. As a consequence, I'm
picking the longest widely accepted line length, 119 characters.
This is also Thomas' preference, because it allows for explicit variable
names, to make the code easier to understand.
120 lines
6.4 KiB
Python
120 lines
6.4 KiB
Python
from transformers import (
|
|
AutoTokenizer,
|
|
AutoConfig,
|
|
AutoModel,
|
|
AutoModelWithLMHead,
|
|
AutoModelForSequenceClassification,
|
|
AutoModelForQuestionAnswering,
|
|
)
|
|
from transformers.file_utils import add_start_docstrings
|
|
|
|
dependencies = ["torch", "tqdm", "boto3", "requests", "regex", "sentencepiece", "sacremoses"]
|
|
|
|
|
|
@add_start_docstrings(AutoConfig.__doc__)
|
|
def config(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased') # Download configuration from S3 and cache.
|
|
config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/') # E.g. config (or model) was saved using `save_pretrained('./test/saved_model/')`
|
|
config = torch.hub.load('huggingface/transformers', 'config', './test/bert_saved_model/my_configuration.json')
|
|
config = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False)
|
|
assert config.output_attention == True
|
|
config, unused_kwargs = torch.hub.load('huggingface/transformers', 'config', 'bert-base-uncased', output_attention=True, foo=False, return_unused_kwargs=True)
|
|
assert config.output_attention == True
|
|
assert unused_kwargs == {'foo': False}
|
|
|
|
"""
|
|
|
|
return AutoConfig.from_pretrained(*args, **kwargs)
|
|
|
|
|
|
@add_start_docstrings(AutoTokenizer.__doc__)
|
|
def tokenizer(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', 'bert-base-uncased') # Download vocabulary from S3 and cache.
|
|
tokenizer = torch.hub.load('huggingface/transformers', 'tokenizer', './test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`
|
|
|
|
"""
|
|
|
|
return AutoTokenizer.from_pretrained(*args, **kwargs)
|
|
|
|
|
|
@add_start_docstrings(AutoModel.__doc__)
|
|
def model(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased') # Download model and configuration from S3 and cache.
|
|
model = torch.hub.load('huggingface/transformers', 'model', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
|
|
model = torch.hub.load('huggingface/transformers', 'model', 'bert-base-uncased', output_attention=True) # Update configuration during loading
|
|
assert model.config.output_attention == True
|
|
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
|
|
config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
|
|
model = torch.hub.load('huggingface/transformers', 'model', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
|
|
|
|
"""
|
|
|
|
return AutoModel.from_pretrained(*args, **kwargs)
|
|
|
|
|
|
@add_start_docstrings(AutoModelWithLMHead.__doc__)
|
|
def modelWithLMHead(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased') # Download model and configuration from S3 and cache.
|
|
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
|
|
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased', output_attention=True) # Update configuration during loading
|
|
assert model.config.output_attention == True
|
|
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
|
|
config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
|
|
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
|
|
|
|
"""
|
|
return AutoModelWithLMHead.from_pretrained(*args, **kwargs)
|
|
|
|
|
|
@add_start_docstrings(AutoModelForSequenceClassification.__doc__)
|
|
def modelForSequenceClassification(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased') # Download model and configuration from S3 and cache.
|
|
model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
|
|
model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', 'bert-base-uncased', output_attention=True) # Update configuration during loading
|
|
assert model.config.output_attention == True
|
|
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
|
|
config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
|
|
model = torch.hub.load('huggingface/transformers', 'modelForSequenceClassification', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
|
|
|
|
"""
|
|
|
|
return AutoModelForSequenceClassification.from_pretrained(*args, **kwargs)
|
|
|
|
|
|
@add_start_docstrings(AutoModelForQuestionAnswering.__doc__)
|
|
def modelForQuestionAnswering(*args, **kwargs):
|
|
r"""
|
|
# Using torch.hub !
|
|
import torch
|
|
|
|
model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased') # Download model and configuration from S3 and cache.
|
|
model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
|
|
model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', 'bert-base-uncased', output_attention=True) # Update configuration during loading
|
|
assert model.config.output_attention == True
|
|
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
|
|
config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
|
|
model = torch.hub.load('huggingface/transformers', 'modelForQuestionAnswering', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
|
|
|
|
"""
|
|
return AutoModelForQuestionAnswering.from_pretrained(*args, **kwargs)
|