mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-15 18:48:24 +06:00
43 lines
2.7 KiB
Python
43 lines
2.7 KiB
Python
__version__ = "1.0.0"
|
|
from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer
|
|
from .tokenization_openai import OpenAIGPTTokenizer
|
|
from .tokenization_transfo_xl import (TransfoXLTokenizer, TransfoXLCorpus)
|
|
from .tokenization_gpt2 import GPT2Tokenizer
|
|
from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE
|
|
from .tokenization_xlm import XLMTokenizer
|
|
from .tokenization_utils import (PreTrainedTokenizer, clean_up_tokenization)
|
|
|
|
from .modeling_bert import (BertConfig, BertModel, BertForPreTraining,
|
|
BertForMaskedLM, BertForNextSentencePrediction,
|
|
BertForSequenceClassification, BertForMultipleChoice,
|
|
BertForTokenClassification, BertForQuestionAnswering,
|
|
load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP,
|
|
BERT_PRETRAINED_CONFIG_ARCHIVE_MAP)
|
|
from .modeling_openai import (OpenAIGPTConfig, OpenAIGPTModel,
|
|
OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel,
|
|
load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
|
OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP)
|
|
from .modeling_transfo_xl import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel,
|
|
load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
|
TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP)
|
|
from .modeling_gpt2 import (GPT2Config, GPT2Model,
|
|
GPT2LMHeadModel, GPT2DoubleHeadsModel,
|
|
load_tf_weights_in_gpt2, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
|
GPT2_PRETRAINED_MODEL_ARCHIVE_MAP)
|
|
from .modeling_xlnet import (XLNetConfig,
|
|
XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel,
|
|
XLNetForSequenceClassification, XLNetForQuestionAnswering,
|
|
load_tf_weights_in_xlnet, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
|
XLNET_PRETRAINED_MODEL_ARCHIVE_MAP)
|
|
from .modeling_xlm import (XLMConfig, XLMModel,
|
|
XLMWithLMHeadModel, XLMForSequenceClassification,
|
|
XLMForQuestionAnswering, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP,
|
|
XLM_PRETRAINED_MODEL_ARCHIVE_MAP)
|
|
from .modeling_utils import (WEIGHTS_NAME, CONFIG_NAME, TF_WEIGHTS_NAME,
|
|
PretrainedConfig, PreTrainedModel, prune_layer, Conv1D)
|
|
|
|
from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule,
|
|
WarmupCosineWithHardRestartsSchedule, WarmupLinearSchedule)
|
|
|
|
from .file_utils import (PYTORCH_PRETRAINED_BERT_CACHE, cached_path)
|