mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-31 02:02:21 +06:00
Merge branch 'master' of https://github.com/huggingface/pytorch-pretrained-BERT
This commit is contained in:
thomwolf
commit
a4086c5de5
@ -4,26 +4,72 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# TensorFlow code"
|
||||
"# Comparing TensorFlow (original) and PyTorch models\n",
|
||||
"\n",
|
||||
"We use this small notebook to test the conversion of the model's weights and to make sure both the TensorFlow and PyTorch are coherent. In particular, we compare the weights of the last layer on a simple example (in `input.txt`).\n",
|
||||
"\n",
|
||||
"To run this notebook, please make sure that your Python environment has both TensorFlow and PyTorch.\n",
|
||||
"You should follow the instructions in the `README.md` and make sure that you have:\n",
|
||||
"- the original TensorFlow implementation\n",
|
||||
"- the `BERT-base, Uncased` model\n",
|
||||
"- run the script `convert_tf_checkpoint_to_pytorch.py` to convert the weights to PyTorch\n",
|
||||
"\n",
|
||||
"Please modify the relative paths accordingly (at the beggining of Sections 1 and 2)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 1/ TensorFlow code"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"original_tf_inplem_dir = \"../bert/\"\n",
|
||||
"model_dir = \"../uncased_L-12_H-768_A-12/\"\n",
|
||||
"\n",
|
||||
"vocab_file = model_dir + \"vocab.txt\"\n",
|
||||
"bert_config_file = model_dir + \"bert_config.json\"\n",
|
||||
"init_checkpoint = model_dir + \"bert_model.ckpt\"\n",
|
||||
"\n",
|
||||
"input_file = \"input.txt\"\n",
|
||||
"max_seq_length = 128"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:37.498678Z",
|
||||
"start_time": "2018-11-03T02:09:36.366672Z"
|
||||
}
|
||||
},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/usr/local/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
|
||||
" from ._conv import register_converters as _register_converters\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import sys\n",
|
||||
"sys.path.append(original_tf_inplem_dir)\n",
|
||||
"\n",
|
||||
"from extract_features import *"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": 3,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:37.621865Z",
|
||||
@ -45,13 +91,6 @@
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"data_dir=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/data/glue_data/MRPC/\"\n",
|
||||
"vocab_file=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/vocab.txt\"\n",
|
||||
"bert_config_file=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/bert_config.json\"\n",
|
||||
"init_checkpoint=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/bert_model.ckpt\"\n",
|
||||
"max_seq_length=128\n",
|
||||
"input_file=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/pytorch-pretrained-BERT/input.txt\"\n",
|
||||
"\n",
|
||||
"layer_indexes = list(range(12))\n",
|
||||
"bert_config = modeling.BertConfig.from_json_file(bert_config_file)\n",
|
||||
"tokenizer = tokenization.FullTokenizer(\n",
|
||||
@ -67,7 +106,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": 4,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:40.831618Z",
|
||||
@ -79,15 +118,15 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x12b0bcc80>) includes params argument, but params are not passed to Estimator.\n",
|
||||
"WARNING:tensorflow:Using temporary folder as model directory: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmpgpb5nz3u\n",
|
||||
"INFO:tensorflow:Using config: {'_model_dir': '/var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmpgpb5nz3u', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
|
||||
"WARNING:tensorflow:Estimator's model_fn (<function model_fn_builder.<locals>.model_fn at 0x1289c1a60>) includes params argument, but params are not passed to Estimator.\n",
|
||||
"WARNING:tensorflow:Using temporary folder as model directory: /var/folders/y2/py87pn6115bdsdftbc6394nh0000gn/T/tmpmcfk2tyr\n",
|
||||
"INFO:tensorflow:Using config: {'_model_dir': '/var/folders/y2/py87pn6115bdsdftbc6394nh0000gn/T/tmpmcfk2tyr', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
|
||||
"graph_options {\n",
|
||||
" rewrite_options {\n",
|
||||
" meta_optimizer_iterations: ONE\n",
|
||||
" }\n",
|
||||
"}\n",
|
||||
", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x12e1160f0>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=2, num_shards=1, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
|
||||
", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': None, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x12c242470>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1, '_tpu_config': TPUConfig(iterations_per_loop=2, num_shards=1, num_cores_per_replica=None, per_host_input_for_training=3, tpu_job_name=None, initial_infeed_sleep_secs=None, input_partition_dims=None), '_cluster': None}\n",
|
||||
"WARNING:tensorflow:Setting TPUConfig.num_shards==1 is an unsupported behavior. Please fix as soon as possible (leaving num_shards as None.\n",
|
||||
"INFO:tensorflow:_TPUContext: eval_on_tpu True\n",
|
||||
"WARNING:tensorflow:eval_on_tpu ignored because use_tpu is False.\n"
|
||||
@ -123,7 +162,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": 5,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:46.413197Z",
|
||||
@ -135,7 +174,7 @@
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"INFO:tensorflow:Could not find trained model in model_dir: /var/folders/yx/cw8n_njx3js5jksyw_qlp8p00000gn/T/tmpgpb5nz3u, running initialization to predict.\n",
|
||||
"INFO:tensorflow:Could not find trained model in model_dir: /var/folders/y2/py87pn6115bdsdftbc6394nh0000gn/T/tmpmcfk2tyr, running initialization to predict.\n",
|
||||
"INFO:tensorflow:Calling model_fn.\n",
|
||||
"INFO:tensorflow:Running infer on CPU\n",
|
||||
"INFO:tensorflow:Done calling model_fn.\n",
|
||||
@ -186,7 +225,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:46.460128Z",
|
||||
@ -211,7 +250,7 @@
|
||||
"(128, 768)"
|
||||
]
|
||||
},
|
||||
"execution_count": 5,
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -227,7 +266,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:46.498637Z",
|
||||
@ -243,12 +282,12 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# PyTorch code"
|
||||
"## 2/ PyTorch code"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:46.660303Z",
|
||||
@ -263,12 +302,22 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"init_checkpoint_pt = \"../pytorch_model/uncased_L-12_H-768_A-12/pytorch_model.bin\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:48.292135Z",
|
||||
"start_time": "2018-11-03T02:09:46.661921Z"
|
||||
}
|
||||
},
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
@ -569,14 +618,12 @@
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 8,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"init_checkpoint_pt=\"/Users/thomaswolf/Documents/Thomas/Code/HF/BERT/google_models/uncased_L-12_H-768_A-12/pytorch_model.bin\"\n",
|
||||
"\n",
|
||||
"device = torch.device(\"cpu\")\n",
|
||||
"model = extract_features_pytorch.BertModel(bert_config)\n",
|
||||
"model.load_state_dict(torch.load(init_checkpoint_pt, map_location='cpu'))\n",
|
||||
@ -585,7 +632,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 11,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:48.332982Z",
|
||||
@ -892,7 +939,7 @@
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -912,7 +959,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 12,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:54.371188Z",
|
||||
@ -1000,7 +1047,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 13,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:57.139854Z",
|
||||
@ -1026,7 +1073,7 @@
|
||||
"(128, 768)"
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -1043,7 +1090,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 14,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:59.000058Z",
|
||||
@ -1068,7 +1115,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"execution_count": 15,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:09:59.462123Z",
|
||||
@ -1090,9 +1137,16 @@
|
||||
"print(tensorflow_outputs[1].shape)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 3/ Comparing the standard deviation on the last layer of both models"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:10:00.014784Z",
|
||||
@ -1106,7 +1160,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"ExecuteTime": {
|
||||
"end_time": "2018-11-03T02:10:09.582557Z",
|
||||
@ -1127,7 +1181,7 @@
|
||||
"4.1671223e-07"
|
||||
]
|
||||
},
|
||||
"execution_count": 24,
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@ -1137,21 +1191,14 @@
|
||||
"print(np.array(tensorflow_outputs[i]).shape, np.array(pytorch_outputs[i]).shape)\n",
|
||||
"np.sqrt(np.mean((np.array(tensorflow_outputs[i]) - np.array(pytorch_outputs[i]))**2.0))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"hide_input": false,
|
||||
"kernelspec": {
|
||||
"display_name": "Python [conda env:bert]",
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "conda-env-bert-py"
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
@ -1163,7 +1210,7 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.7"
|
||||
"version": "3.6.5"
|
||||
},
|
||||
"toc": {
|
||||
"colors": {
|
||||
|
||||
802
README.md
802
README.md
@ -1,9 +1,11 @@
|
||||
# BERT
|
||||
# PyTorch implementation of Google AI's BERT
|
||||
|
||||
|
||||
## Introduction
|
||||
|
||||
This is a PyTorch implementation of the [TensorFlow code](https://github.com/google-research/bert) released by Google AI with the paper [BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding](https://arxiv.org/abs/1810.04805).
|
||||
|
||||
|
||||
## Converting the TensorFlow pre-trained models to Pytorch
|
||||
|
||||
You can convert the pre-trained weights released by GoogleAI by calling the script `convert_tf_checkpoint_to_pytorch.py`.
|
||||
@ -21,6 +23,7 @@ python convert_tf_checkpoint_to_pytorch.py \
|
||||
--pytorch_dump_path=$BERT_PYTORCH_DIR/pytorch_model.bin
|
||||
```
|
||||
|
||||
|
||||
## Fine-tuning with BERT: running the examples
|
||||
|
||||
We showcase the same examples as in the original implementation: fine-tuning on the MRPC classification corpus and the question answering dataset SQUAD.
|
||||
@ -53,6 +56,7 @@ python run_classifier_pytorch.py \
|
||||
--output_dir /tmp/mrpc_output_pytorch/
|
||||
```
|
||||
|
||||
The next example fine-tunes `BERT-Base` on the SQuAD question answering task.
|
||||
|
||||
The data for SQuAD can be downloaded with the following links and should be saved in a `$SQUAD_DIR` directory.
|
||||
* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json)
|
||||
@ -62,6 +66,7 @@ The data for SQuAD can be downloaded with the following links and should be save
|
||||
|
||||
```shell
|
||||
export SQUAD_DIR=/path/to/SQUAD
|
||||
|
||||
python run_squad_pytorch.py \
|
||||
--vocab_file=$BERT_BASE_DIR/vocab.txt \
|
||||
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
|
||||
@ -75,797 +80,26 @@ python run_squad_pytorch.py \
|
||||
--num_train_epochs=2.0 \
|
||||
--max_seq_length=384 \
|
||||
--doc_stride=128 \
|
||||
--output_dir=/tmp/squad_base_pytorch/
|
||||
--output_dir=../debug_squad/
|
||||
```
|
||||
|
||||
|
||||
## Comparing TensorFlow and PyTorch models
|
||||
|
||||
We also include [a small Notebook](https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/Comparing%20TF%20and%20PT%20models.ipynb) we used to verify that the conversion of the weights to PyTorch are consistent with the original TensorFlow weights.
|
||||
Please follow the instructions in the Notebook to run it.
|
||||
|
||||
|
||||
## Introduction
|
||||
## Note on pre-training
|
||||
|
||||
**BERT**, or **B**idirectional **E**mbedding **R**epresentations from
|
||||
**T**ransformers, is a new method of pre-training language representations which
|
||||
obtains state-of-the-art results on a wide array of Natural Language Processing
|
||||
(NLP) tasks.
|
||||
The original TensorFlow code also release two scripts for pre-training BERT: [create_pretraining_data.py](https://github.com/google-research/bert/blob/master/create_pretraining_data.py) and [run_pretraining.py](https://github.com/google-research/bert/blob/master/run_pretraining.py).
|
||||
As the authors notice, pre-training BERT is particularly expensive and requires TPU to run in a reasonable amout of time (see [here](https://github.com/google-research/bert#pre-training-with-bert)).
|
||||
|
||||
Our academic paper which describes BERT in detail and provides full results on a
|
||||
number of tasks can be found here:
|
||||
[https://arxiv.org/abs/1810.04805](https://arxiv.org/abs/1810.04805).
|
||||
We have decided **not** to port these scripts for now and wait for the TPU support on PyTorch (see the recent [official announcement](https://cloud.google.com/blog/products/ai-machine-learning/introducing-pytorch-across-google-cloud)).
|
||||
|
||||
To give a few numbers, here are the results on the
|
||||
[SQuAD v1.1](https://rajpurkar.github.io/SQuAD-explorer/) question answering
|
||||
task:
|
||||
|
||||
SQuAD v1.1 Leaderboard (Oct 8th 2018) | Test EM | Test F1
|
||||
------------------------------------- | :------: | :------:
|
||||
1st Place Ensemble - BERT | **87.4** | **93.2**
|
||||
2nd Place Ensemble - nlnet | 86.0 | 91.7
|
||||
1st Place Single Model - BERT | **85.1** | **91.8**
|
||||
2nd Place Single Model - nlnet | 83.5 | 90.1
|
||||
## Requirements
|
||||
|
||||
And several natural language inference tasks:
|
||||
|
||||
System | MultiNLI | Question NLI | SWAG
|
||||
----------------------- | :------: | :----------: | :------:
|
||||
BERT | **86.7** | **91.1** | **86.3**
|
||||
OpenAI GPT (Prev. SOTA) | 82.2 | 88.1 | 75.0
|
||||
|
||||
Plus many other tasks.
|
||||
|
||||
Moreover, these results were all obtained with almost no task-specific neural
|
||||
network architecture design.
|
||||
|
||||
If you already know what BERT is and you just want to get started, you can
|
||||
[download the pre-trained models](#pre-trained-models) and
|
||||
[run a state-of-the-art fine-tuning](#fine-tuning-with-bert) in only a few
|
||||
minutes.
|
||||
|
||||
## What is BERT?
|
||||
|
||||
BERT is method of pre-training language representations, meaning that we train a
|
||||
general-purpose "language understanding" model on a large text corpus (like
|
||||
Wikipedia), and then use that model for downstream NLP tasks that we are about
|
||||
(like question answering). BERT outperforms previous methods because it is the
|
||||
first *unsupervised*, *deeply bidirectional* system for pre-training NLP.
|
||||
|
||||
*Unsupervised* means that BERT was trained using only a plain text corpus, which
|
||||
is important because an enormous amount of plain text data is publicly available
|
||||
on the web in many languages.
|
||||
|
||||
Pre-trained representations can also either be *context-free* or *contextual*,
|
||||
and contextual representations can further be *unidirectional* or
|
||||
*bidirectional*. Context-free models such as
|
||||
[word2vec](https://www.tensorflow.org/tutorials/representation/word2vec) or
|
||||
[GloVe](https://nlp.stanford.edu/projects/glove/) generate a single "word
|
||||
embedding" representation for each word in the vocabulary, so `bank` would have
|
||||
the same representation in `bank deposit` and `river bank`. Contextual models
|
||||
instead generate a representation of each word that is based on the other words
|
||||
in the sentence.
|
||||
|
||||
BERT was built upon recent work in pre-training contextual representations —
|
||||
including [Semi-supervised Sequence Learning](https://arxiv.org/abs/1511.01432),
|
||||
[Generative Pre-Training](https://blog.openai.com/language-unsupervised/),
|
||||
[ELMo](https://allennlp.org/elmo), and
|
||||
[ULMFit](http://nlp.fast.ai/classification/2018/05/15/introducting-ulmfit.html)
|
||||
— but crucially these models are all *unidirectional* or *shallowly
|
||||
bidirectional*. This means that each word is only contextualized using the words
|
||||
to its left (or right). For example, in the sentence `I made a bank deposit` the
|
||||
unidirectional representation of `bank` is only based on `I made a` but not
|
||||
`deposit`. Some previous work does combine the representations from separate
|
||||
left-context and right-context models, but only in a "shallow" manner. BERT
|
||||
represents "bank" using both its left and right context — `I made a ... deposit`
|
||||
— starting from the very bottom of a deep neural network, so it is *deeply
|
||||
bidirectional*.
|
||||
|
||||
BERT uses a simple approach for this: We mask out 15% of the words in the input,
|
||||
run the entire sequence through a deep bidirectional
|
||||
[Transformer](https://arxiv.org/abs/1706.03762) encoder, and then predict only
|
||||
the masked words. For example:
|
||||
|
||||
```
|
||||
Input: the man went to the [MASK1] . he bought a [MASK2] of milk.
|
||||
Labels: [MASK1] = store; [MASK2] = gallon
|
||||
```
|
||||
|
||||
In order to learn relationships between sentences, we also train on a simple
|
||||
task which can be generated from any monolingual corpus: Given two sentences `A`
|
||||
and `B`, is `B` the actual next sentence that comes after `A`, or just a random
|
||||
sentence from the corpus?
|
||||
|
||||
```
|
||||
Sentence A: the man went to the store .
|
||||
Sentence B: he bought a gallon of milk .
|
||||
Label: IsNextSentence
|
||||
```
|
||||
|
||||
```
|
||||
Sentence A: the man went to the store .
|
||||
Sentence B: penguins are flightless .
|
||||
Label: NotNextSentence
|
||||
```
|
||||
|
||||
We then train a large model (12-layer to 24-layer Transformer) on a large corpus
|
||||
(Wikipedia + [BookCorpus](http://yknzhu.wixsite.com/mbweb)) for a long time (1M
|
||||
update steps), and that's BERT.
|
||||
|
||||
Using BERT has two stages: *Pre-training* and *fine-tuning*.
|
||||
|
||||
**Pre-training** is fairly expensive (four days on 4 to 16 Cloud TPUs), but is a
|
||||
one-time procedure for each language (current models are English-only, but
|
||||
multilingual models will be released in the near future). We are releasing a
|
||||
number of pre-trained models from the paper which were pre-trained at Google.
|
||||
Most NLP researchers will never need to pre-train their own model from scratch.
|
||||
|
||||
**Fine-tuning** is inexpensive. All of the results in the paper can be
|
||||
replicated in at most 1 hour on a single Cloud TPU, or a few hours on a GPU,
|
||||
starting from the exact same pre-trained model. SQuAD, for example, can be
|
||||
trained in around 30 minutes on a single Cloud TPU to achieve a Dev F1 score of
|
||||
91.0%, which is the single system state-of-the-art.
|
||||
|
||||
The other important aspect of BERT is that it can be adapted to many types of
|
||||
NLP tasks very easily. In the paper, we demonstrate state-of-the-art results on
|
||||
sentence-level (e.g., SST-2), sentence-pair-level (e.g., MultiNLI), word-level
|
||||
(e.g., NER), and span-level (e.g., SQuAD) tasks with almost no task-specific
|
||||
modifications.
|
||||
|
||||
## What has been released in this repository?
|
||||
|
||||
We are releasing the following:
|
||||
|
||||
* TensorFlow code for the BERT model architecture (which is mostly a standard
|
||||
[Transformer](https://arxiv.org/abs/1706.03762) architecture).
|
||||
* Pre-trained checkpoints for both the lowercase and cased version of
|
||||
`BERT-Base` and `BERT-Large` from the paper.
|
||||
* TensorFlow code for push-button replication of the most important
|
||||
fine-tuning experiments from the paper, including SQuAD, MultiNLI, and MRPC.
|
||||
|
||||
All of the code in this repository works out-of-the-box with CPU, GPU, and Cloud
|
||||
TPU.
|
||||
|
||||
## Pre-trained models
|
||||
|
||||
We are releasing the `BERT-Base` and `BERT-Large` models from the paper.
|
||||
`Uncased` means that the text has been lowercased before WordPiece tokenization,
|
||||
e.g., `John Smith` becomes `john smith`. The `Uncased` model also strips out any
|
||||
accent markers. `Cased` means that the true case and accent markers are
|
||||
preserved. Typically, the `Uncased` model is better unless you know that case
|
||||
information is important for your task (e.g., Named Entity Recognition or
|
||||
Part-of-Speech tagging).
|
||||
|
||||
These models are all released under the same license as the source code (Apache
|
||||
2.0).
|
||||
|
||||
The links to the models are here (right-cick, 'Save link as...' on the name):
|
||||
|
||||
* **[`BERT-Base, Uncased`](https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-12_H-768_A-12.zip)**:
|
||||
12-layer, 768-hidden, 12-heads, 110M parameters
|
||||
* **[`BERT-Large, Uncased`](https://storage.googleapis.com/bert_models/2018_10_18/uncased_L-24_H-1024_A-16.zip)**:
|
||||
24-layer, 1024-hidden, 16-heads, 340M parameters
|
||||
* **[`BERT-Base, Cased`](https://storage.googleapis.com/bert_models/2018_10_18/cased_L-12_H-768_A-12.zip)**:
|
||||
12-layer, 768-hidden, 12-heads , 110M parameters
|
||||
* **`BERT-Large, Cased`**: 24-layer, 1024-hidden, 16-heads, 340M parameters
|
||||
(Not available yet. Needs to be re-generated).
|
||||
|
||||
Each .zip file contains three items:
|
||||
|
||||
* A TensorFlow checkpoint (`bert_model.ckpt`) containing the pre-trained
|
||||
weights (which is actually 3 files).
|
||||
* A vocab file (`vocab.txt`) to map WordPiece to word id.
|
||||
* A config file (`bert_config.json`) which specifies the hyperparameters of
|
||||
the model.
|
||||
|
||||
## Fine-tuning with BERT
|
||||
|
||||
**Important**: All results on the paper were fine-tuned on a single Cloud TPU,
|
||||
which has 64GB of RAM. It is currently not possible to re-produce most of the
|
||||
`BERT-Large` results on the paper using a GPU with 12GB - 16GB of RAM, because
|
||||
the maximum batch size that can fit in memory is too small. We are working on
|
||||
adding code to this repository which allows for much larger effective batch size
|
||||
on the GPU. See the section on [out-of-memory issues](#out-of-memory-issues) for
|
||||
more details.
|
||||
|
||||
This code was tested with TensorFlow 1.11.0. It was tested with Python2 and
|
||||
Python3 (but more thoroughly with Python2, since this is what's used internally
|
||||
in Google).
|
||||
|
||||
The fine-tuning examples which use `BERT-Base` should be able to run on a GPU
|
||||
that has at least 12GB of RAM using the hyperparameters given.
|
||||
|
||||
### Fine-tuning with Cloud TPUs
|
||||
|
||||
Most of the examples below assumes that you will be running training/evaluation
|
||||
on your local machine, using a GPU like a Titan X or GTX 1080.
|
||||
|
||||
However, if you have access to a Cloud TPU that you want to train on, just add
|
||||
the following flags to `run_classifier.py` or `run_squad.py`:
|
||||
|
||||
```
|
||||
--use_tpu=True \
|
||||
--tpu_name=$TPU_NAME
|
||||
```
|
||||
|
||||
Please see the
|
||||
[Google Cloud TPU tutorial](https://cloud.google.com/tpu/docs/tutorials/mnist)
|
||||
for how to use Cloud TPUs.
|
||||
|
||||
On Cloud TPUs, the pretrained model and the output directory will need to be on
|
||||
Google Cloud Storage. For example, if you have a bucket named `some_bucket`, you
|
||||
might use the following flags instead:
|
||||
|
||||
```
|
||||
--output_dir=gs://some_bucket/my_output_dir/
|
||||
```
|
||||
|
||||
The unzipped pre-trained model files can also be found in the Google Cloud
|
||||
Storage folder `gs://bert_models/2018_10_18`. For example:
|
||||
|
||||
```
|
||||
export BERT_BASE_DIR=gs://bert_models/2018_10_18/uncased_L-12_H-768_A-12
|
||||
```
|
||||
|
||||
### Sentence (and sentence-pair) classification tasks
|
||||
|
||||
Before running this example you must download the
|
||||
[GLUE data](https://gluebenchmark.com/tasks) by running
|
||||
[this script](https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e)
|
||||
and unpack it to some directory `$GLUE_DIR`. Next, download the `BERT-Base`
|
||||
checkpoint and unzip it to some directory `$BERT_BASE_DIR`.
|
||||
|
||||
This example code fine-tunes `BERT-Base` on the Microsoft Research Paraphrase
|
||||
Corpus (MRPC) corpus, which only contains 3,600 examples and can fine-tune in a
|
||||
few minutes on most GPUs.
|
||||
|
||||
```shell
|
||||
export BERT_BASE_DIR=/path/to/bert/uncased_L-12_H-768_A-12
|
||||
export GLUE_DIR=/path/to/glue
|
||||
|
||||
python run_classifier.py \
|
||||
--task_name=MRPC \
|
||||
--do_train=true \
|
||||
--do_eval=true \
|
||||
--data_dir=$GLUE_DIR/MRPC \
|
||||
--vocab_file=$BERT_BASE_DIR/vocab.txt \
|
||||
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
|
||||
--init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \
|
||||
--max_seq_length=128 \
|
||||
--train_batch_size=32 \
|
||||
--learning_rate=2e-5 \
|
||||
--num_train_epochs=3.0 \
|
||||
--output_dir=/tmp/mrpc_output/
|
||||
```
|
||||
|
||||
You should see output like this:
|
||||
|
||||
```
|
||||
***** Eval results *****
|
||||
eval_accuracy = 0.845588
|
||||
eval_loss = 0.505248
|
||||
global_step = 343
|
||||
loss = 0.505248
|
||||
```
|
||||
|
||||
This means that the Dev set accuracy was 84.55%. Small sets like MRPC have a
|
||||
high variance in the Dev set accuracy, even when starting from the same
|
||||
pre-training checkpoint. If you re-run multiple times (making sure to point to
|
||||
different `output_dir`), you should see results between 84% and 88%.
|
||||
|
||||
A few other pre-trained models are implemented off-the-shelf in
|
||||
`run_classifier.py`, so it should be straightforward to follow those examples to
|
||||
use BERT for any single-sentence or sentence-pair classification task.
|
||||
|
||||
Note: You might see a message `Running train on CPU`. This really just means
|
||||
that it's running on something other than a Cloud TPU, which includes a GPU.
|
||||
|
||||
### SQuAD
|
||||
|
||||
The Stanford Question Answering Dataset (SQuAD) is a popular question answering
|
||||
benchmark dataset. BERT (at the time of the release) obtains state-of-the-art
|
||||
results on SQuAD with almost no task-specific network architecture modifications
|
||||
or data augmentation. However, it does require semi-complex data pre-processing
|
||||
and post-processing to deal with (a) the variable-length nature of SQuAD context
|
||||
paragraphs, and (b) the character-level answer annotations which are used for
|
||||
SQuAD training. This processing is implemented and documented in `run_squad.py`.
|
||||
|
||||
To run on SQuAD, you will first need to download the dataset. The
|
||||
[SQuAD website](https://rajpurkar.github.io/SQuAD-explorer/) does not seem to
|
||||
link to the v1.1 datasets any longer, but the necessary files can be found here:
|
||||
|
||||
* [train-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v1.1.json)
|
||||
* [dev-v1.1.json](https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json)
|
||||
* [evaluate-v1.1.py](https://github.com/allenai/bi-att-flow/blob/master/squad/evaluate-v1.1.py)
|
||||
|
||||
Download these to some directory `$SQUAD_DIR`.
|
||||
|
||||
The state-of-the-art SQuAD results from the paper currently cannot be reproduced
|
||||
on a 12GB-16GB GPU due to memory constraints (in fact, even batch size 1 does
|
||||
not seem to fit on a 12GB GPU using `BERT-Large`). However, a reasonably strong
|
||||
`BERT-Base` model can be trained on the GPU with these hyperparameters:
|
||||
|
||||
```shell
|
||||
python run_squad.py \
|
||||
--vocab_file=$BERT_BASE_DIR/vocab.txt \
|
||||
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
|
||||
--init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \
|
||||
--do_train=True \
|
||||
--train_file=$SQUAD_DIR/train-v1.1.json \
|
||||
--do_predict=True \
|
||||
--predict_file=$SQUAD_DIR/dev-v1.1.json \
|
||||
--train_batch_size=12 \
|
||||
--learning_rate=5e-5 \
|
||||
--num_train_epochs=2.0 \
|
||||
--max_seq_length=384 \
|
||||
--doc_stride=128 \
|
||||
--output_dir=/tmp/squad_base/
|
||||
```
|
||||
|
||||
The dev set predictions will be saved into a file called `predictions.json` in
|
||||
the `output_dir`:
|
||||
|
||||
```shell
|
||||
python $SQUAD_DIR/evaluate-v1.1.py $SQUAD_DIR/dev-v1.1.json ./squad/predictions.json
|
||||
```
|
||||
|
||||
Which should produce an output like this:
|
||||
|
||||
```shell
|
||||
{"f1": 88.41249612335034, "exact_match": 81.2488174077578}
|
||||
```
|
||||
|
||||
You should see a result similar to the 88.5% reported in the paper for
|
||||
`BERT-Base`.
|
||||
|
||||
If you have access to a Cloud TPU, you can train with `BERT-Large`. Here is a
|
||||
set of hyperparameters (slightly different than the paper) which consistently
|
||||
obtain around 90.5%-91.0% F1 single-system trained only on SQuAD:
|
||||
|
||||
```shell
|
||||
python run_squad.py \
|
||||
--vocab_file=$BERT_LARGE_DIR/vocab.txt \
|
||||
--bert_config_file=$BERT_LARGE_DIR/bert_config.json \
|
||||
--init_checkpoint=$BERT_LARGE_DIR/bert_model.ckpt \
|
||||
--do_train=True \
|
||||
--train_file=$SQUAD_DIR/train-v1.1.json \
|
||||
--do_predict=True \
|
||||
--predict_file=$SQUAD_DIR/dev-v1.1.json \
|
||||
--train_batch_size=48 \
|
||||
--learning_rate=5e-5 \
|
||||
--num_train_epochs=2.0 \
|
||||
--max_seq_length=384 \
|
||||
--doc_stride=128 \
|
||||
--output_dir=gs://some_bucket/squad_large/ \
|
||||
--use_tpu=True \
|
||||
--tpu_name=$TPU_NAME
|
||||
```
|
||||
|
||||
For example, one random run with these parameters produces the following Dev
|
||||
scores:
|
||||
|
||||
```shell
|
||||
{"f1": 90.87081895814865, "exact_match": 84.38978240302744}
|
||||
```
|
||||
|
||||
If you fine-tune for one epoch on
|
||||
[TriviaQA](http://nlp.cs.washington.edu/triviaqa/) before this the results will
|
||||
be even better, but you will need to convert TriviaQA into the SQuAD json
|
||||
format.
|
||||
|
||||
### Out-of-memory issues
|
||||
|
||||
All experiments in the paper were fine-tuned on a Cloud TPU, which has 64GB of
|
||||
device RAM. Therefore, when using a GPU with 12GB - 16GB of RAM, you are likely
|
||||
to encounter out-of-memory issues if you use the same hyperparameters described
|
||||
in the paper.
|
||||
|
||||
The factors that affect memory usage are:
|
||||
|
||||
* **`max_seq_length`**: The released models were trained with sequence lengths
|
||||
up to 512, but you can fine-tune with a shorter max sequence length to save
|
||||
substantial memory. This is controlled by the `max_seq_length` flag in our
|
||||
example code.
|
||||
|
||||
* **`train_batch_size`**: The memory usage is also directly proportional to
|
||||
the batch size.
|
||||
|
||||
* **Model type, `BERT-Base` vs. `BERT-Large`**: The `BERT-Large` model
|
||||
requires significantly more memory than `BERT-Base`.
|
||||
|
||||
* **Optimizer**: The default optimizer for BERT is Adam, which requires a lot
|
||||
of extra memory to store the `m` and `v` vectors. Switching to a more memory
|
||||
efficient optimizer can reduce memory usage, but can also affect the
|
||||
results. We have not experimented with other optimizers for fine-tuning.
|
||||
|
||||
Using the default training scripts (`run_classifier.py` and `run_squad.py`), we
|
||||
benchmarked the maximum batch size on single Titan X GPU (12GB RAM) with
|
||||
TensorFlow 1.11.0:
|
||||
|
||||
System | Seq Length | Max Batch Size
|
||||
------------ | ---------- | --------------
|
||||
`BERT-Base` | 64 | 64
|
||||
... | 128 | 32
|
||||
... | 256 | 16
|
||||
... | 320 | 14
|
||||
... | 384 | 12
|
||||
... | 512 | 6
|
||||
`BERT-Large` | 64 | 12
|
||||
... | 128 | 6
|
||||
... | 256 | 2
|
||||
... | 320 | 1
|
||||
... | 384 | 0
|
||||
... | 512 | 0
|
||||
|
||||
Unfortunately, these max batch sizes for `BERT-Large` are so small that they
|
||||
will actually harm the model accuracy, regardless of the learning rate used. We
|
||||
are working on adding code to this repository which will allow much larger
|
||||
effective batch sizes to be used on the GPU. The code will be based on one (or
|
||||
both) of the following techniques:
|
||||
|
||||
* **Gradient accumulation**: The samples in a minibatch are typically
|
||||
independent with respect to gradient computation (excluding batch
|
||||
normalization, which is not used here). This means that the gradients of
|
||||
multiple smaller minibatches can be accumulated before performing the weight
|
||||
update, and this will be exactly equivalent to a single larger update.
|
||||
|
||||
* [**Gradient checkpointing**](https://github.com/openai/gradient-checkpointing):
|
||||
The major use of GPU/TPU memory during DNN training is caching the
|
||||
intermediate activations in the forward pass that are necessary for
|
||||
efficient computation in the backward pass. "Gradient checkpointing" trades
|
||||
memory for compute time by re-computing the activations in an intelligent
|
||||
way.
|
||||
|
||||
**However, this is not implemented in the current release.**
|
||||
|
||||
## Using BERT to extract fixed feature vectors (like ELMo)
|
||||
|
||||
In certain cases, rather than fine-tuning the entire pre-trained model
|
||||
end-to-end, it can be beneficial to obtained *pre-trained contextual
|
||||
embeddings*, which are fixed contextual representations of each input token
|
||||
generated from the hidden layers of the pre-trained model. This should also
|
||||
mitigate most of the out-of-memory issues.
|
||||
|
||||
As an example, we include the script `extract_features.py` which can be used
|
||||
like this:
|
||||
|
||||
```shell
|
||||
# Sentence A and Sentence B are separated by the ||| delimiter.
|
||||
# For single sentence inputs, don't use the delimiter.
|
||||
echo 'Who was Jim Henson ? ||| Jim Henson was a puppeteer' > /tmp/input.txt
|
||||
|
||||
python extract_features.py \
|
||||
--input_file=/tmp/input.txt \
|
||||
--output_file=/tmp/output.jsonl \
|
||||
--vocab_file=$BERT_BASE_DIR/vocab.txt \
|
||||
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
|
||||
--init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \
|
||||
--layers=-1,-2,-3,-4 \
|
||||
--max_seq_length=128 \
|
||||
--batch_size=8
|
||||
```
|
||||
|
||||
This will create a JSON file (one line per line of input) containing the BERT
|
||||
activations from each Transformer layer specified by `layers` (-1 is the final
|
||||
hidden layer of the Transformer, etc.)
|
||||
|
||||
Note that this script will produce very large output files (by default, around
|
||||
15kb for every input token).
|
||||
|
||||
If you need to maintain alignment between the original and tokenized words (for
|
||||
projecting training labels), see the [Tokenization](#tokenization) section
|
||||
below.
|
||||
|
||||
## Tokenization
|
||||
|
||||
For sentence-level tasks (or sentence-pair) tasks, tokenization is very simple.
|
||||
Just follow the example code in `run_classifier.py` and `extract_features.py`.
|
||||
The basic procedure for sentence-level tasks is:
|
||||
|
||||
1. Instantiate an instance of `tokenizer = tokenization.FullTokenizer`
|
||||
|
||||
2. Tokenize the raw text with `tokens = tokenizer.tokenize(raw_text)`.
|
||||
|
||||
3. Truncate to the maximum sequence length. (You can use up to 512, but you
|
||||
probably want to use shorter if possible for memory and speed reasons.)
|
||||
|
||||
4. Add the `[CLS]` and `[SEP]` tokens in the right place.
|
||||
|
||||
Word-level and span-level tasks (e.g., SQuAD and NER) are more complex, since
|
||||
you need to maintain alignment between your input text and output text so that
|
||||
you can project your training labels. SQuAD is a particularly complex example
|
||||
because the input labels are *character*-based, and SQuAD paragraphs are often
|
||||
longer than our maximum sequence length. See the code in `run_squad.py` to show
|
||||
how we handle this.
|
||||
|
||||
Before we describe the general recipe for handling word-level tasks, it's
|
||||
important to understand what exactly our tokenizer is doing. It has three main
|
||||
steps:
|
||||
|
||||
1. **Text normalization**: Convert all whitespace characters to spaces, and
|
||||
(for the `Uncased` model) lowercase the input and strip out accent markers.
|
||||
E.g., `John Johanson's, → john johanson's,`.
|
||||
|
||||
2. **Punctuation splitting**: Split *all* punctuation characters on both sides
|
||||
(i.e., add whitespace around all punctuation characters). Punctuation
|
||||
characters are defined as (a) Anything with a `P*` Unicode class, (b) any
|
||||
non-letter/number/space ASCII character (e.g., characters like `$` which are
|
||||
technically not punctuation). E.g., `john johanson's, → john johanson ' s ,`
|
||||
|
||||
3. **WordPiece tokenization**: Apply whitespace tokenization to the output of
|
||||
the above procedure, and apply
|
||||
[WordPiece](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/data_generators/text_encoder.py)
|
||||
tokenization to each token separately. (Our implementation is directly based
|
||||
on the one from `tensor2tensor`, which is linked). E.g., `john johanson ' s
|
||||
, → john johan ##son ' s ,`
|
||||
|
||||
The advantage of this scheme is that it is "compatible" with most existing
|
||||
English tokenizers. For example, imagine that you have a part-of-speech tagging
|
||||
task which looks like this:
|
||||
|
||||
```
|
||||
Input: John Johanson 's house
|
||||
Labels: NNP NNP POS NN
|
||||
```
|
||||
|
||||
The tokenized output will look like this:
|
||||
|
||||
```
|
||||
Tokens: john johan ##son ' s house
|
||||
```
|
||||
|
||||
Crucially, this would be the same output as if the raw text were `John
|
||||
Johanson's house` (with no space before the `'s`).
|
||||
|
||||
If you have a pre-tokenized representation with word-level annotations, you can
|
||||
simply tokenize each input word independently, and deterministically maintain an
|
||||
original-to-tokenized alignment:
|
||||
|
||||
```python
|
||||
### Input
|
||||
orig_tokens = ["John", "Johanson", "'s", "house"]
|
||||
labels = ["NNP", "NNP", "POS", "NN"]
|
||||
|
||||
### Output
|
||||
bert_tokens = []
|
||||
|
||||
# Token map will be an int -> int mapping between the `orig_tokens` index and
|
||||
# the `bert_tokens` index.
|
||||
orig_to_tok_map = []
|
||||
|
||||
tokenizer = tokenization.FullTokenizer(
|
||||
vocab_file=vocab_file, do_lower_case=True)
|
||||
|
||||
bert_tokens.append("[CLS]")
|
||||
for orig_token in orig_tokens:
|
||||
orig_to_tok_map.append(len(bert_tokens))
|
||||
bert_tokens.extend(tokenizer.tokenize(orig_token))
|
||||
bert_tokens.append("[SEP]")
|
||||
|
||||
# bert_tokens == ["[CLS]", "john", "johan", "##son", "'", "s", "house", "[SEP]"]
|
||||
# orig_to_tok_map == [1, 2, 4, 6]
|
||||
```
|
||||
|
||||
Now `orig_to_tok_map` can be used to project `labels` to the tokenized
|
||||
representation.
|
||||
|
||||
There are common English tokenization schemes which will cause a slight mismatch
|
||||
between how BERT was pre-trained. For example, if your input tokenization splits
|
||||
off contractions like `do n't`, this will cause a mismatch. If it is possible to
|
||||
do so, you should pre-process your data to convert these back to raw-looking
|
||||
text, but if it's not possible, this mismatch is likely not a big deal.
|
||||
|
||||
## Pre-training with BERT
|
||||
|
||||
We are releasing code to do "masked LM" and "next sentence prediction" on an
|
||||
arbitrary text corpus. Note that this is *not* the exact code that was used for
|
||||
the paper (the original code was written in C++, and had some additional
|
||||
complexity), but this code does generate pre-training data as described in the
|
||||
paper.
|
||||
|
||||
Here's how to run the data generation. The input is a plain text file, with one
|
||||
sentence per line. (It is important that these be actual sentences for the "next
|
||||
sentence prediction" task). Documents are delimited by empty lines. The output
|
||||
is a set of `tf.train.Example`s serialized into `TFRecord` file format.
|
||||
|
||||
This script stores all of the examples for the entire input file in memory, so
|
||||
for large data files you should shard the input file and call the script
|
||||
multiple times. (You can pass in a file glob to `run_pretraining.py`, e.g.,
|
||||
`tf_examples.tf_record*`.)
|
||||
|
||||
The `max_predictions_per_seq` is the maximum number of masked LM predictions per
|
||||
sequence. You should set this to around `max_seq_length` * `masked_lm_prob` (the
|
||||
script doesn't do that automatically because the exact value needs to be passed
|
||||
to both scripts).
|
||||
|
||||
```shell
|
||||
python create_pretraining_data.py \
|
||||
--input_file=./sample_text.txt \
|
||||
--output_file=/tmp/tf_examples.tfrecord \
|
||||
--vocab_file=$BERT_BASE_DIR/vocab.txt \
|
||||
--do_lower_case=True \
|
||||
--max_seq_length=128 \
|
||||
--max_predictions_per_seq=20 \
|
||||
--masked_lm_prob=0.15 \
|
||||
--random_seed=12345 \
|
||||
--dupe_factor=5
|
||||
```
|
||||
|
||||
Here's how to run the pre-training. Do not include `init_checkpoint` if you are
|
||||
pre-training from scratch. The model configuration (including vocab size) is
|
||||
specified in `bert_config_file`. This demo code only pre-trains for a small
|
||||
number of steps (20), but in practice you will probably want to set
|
||||
`num_train_steps` to 10000 steps or more. The `max_seq_length` and
|
||||
`max_predictions_per_seq` parameters passed to `run_pretraining.py` must be the
|
||||
same as `create_pretraining_data.py`.
|
||||
|
||||
```shell
|
||||
python run_pretraining.py \
|
||||
--input_file=/tmp/tf_examples.tfrecord \
|
||||
--output_dir=/tmp/pretraining_output \
|
||||
--do_train=True \
|
||||
--do_eval=True \
|
||||
--bert_config_file=$BERT_BASE_DIR/bert_config.json \
|
||||
--init_checkpoint=$BERT_BASE_DIR/bert_model.ckpt \
|
||||
--train_batch_size=32 \
|
||||
--max_seq_length=128 \
|
||||
--max_predictions_per_seq=20 \
|
||||
--num_train_steps=20 \
|
||||
--num_warmup_steps=10 \
|
||||
--learning_rate=2e-5
|
||||
```
|
||||
|
||||
This will produce an output like this:
|
||||
|
||||
```
|
||||
***** Eval results *****
|
||||
global_step = 20
|
||||
loss = 0.0979674
|
||||
masked_lm_accuracy = 0.985479
|
||||
masked_lm_loss = 0.0979328
|
||||
next_sentence_accuracy = 1.0
|
||||
next_sentence_loss = 3.45724e-05
|
||||
```
|
||||
|
||||
Note that since our `sample_text.txt` file is very small, this example training
|
||||
will overfit that data in only a few steps and produce unrealistically high
|
||||
accuracy numbers.
|
||||
|
||||
### Pre-training tips and caveats
|
||||
|
||||
* If your task has a large domain-specific corpus available (e.g., "movie
|
||||
reviews" or "scientific papers"), it will likely be beneficial to run
|
||||
additional steps of pre-training on your corpus, starting from the BERT
|
||||
checkpoint.
|
||||
* The learning rate we used in the paper was 1e-4. However, if you are doing
|
||||
additional steps of pre-training starting from an existing BERT checkpoint,
|
||||
you should use a smaller learning rate (e.g., 2e-5).
|
||||
* Current BERT models are English-only, but we do plan to release a
|
||||
multilingual model which has been pre-trained on a lot of languages in the
|
||||
near future (hopefully by the end of November 2018).
|
||||
* Longer sequences are disproportionately expensive because attention is
|
||||
quadratic to the sequence length. In other words, a batch of 64 sequences of
|
||||
length 512 is much more expensive than a batch of 256 sequences of
|
||||
length 128. The fully-connected/convolutional cost is the same, but the
|
||||
attention cost is far greater for the 512-length sequences. Therefore, one
|
||||
good recipe is to pre-train for, say, 90,000 steps with a sequence length of
|
||||
128 and then for 10,000 additional steps with a sequence length of 512. The
|
||||
very long sequences are mostly needed to learn positional embeddings, which
|
||||
can be learned fairly quickly. Note that this does require generating the
|
||||
data twice with different values of `max_seq_length`.
|
||||
* If you are pre-training from scratch, be prepared that pre-training is
|
||||
computationally expensive, especially on GPUs. If you are pre-training from
|
||||
scratch, our recommended recipe is to pre-train a `BERT-Base` on a single
|
||||
[preemptable Cloud TPU v2](https://cloud.google.com/tpu/docs/pricing), which
|
||||
takes about 2 weeks at a cost of about $500 USD (based on the pricing in
|
||||
October 2018). You will have to scale down the batch size when only training
|
||||
on a single Cloud TPU, compared to what was used in the paper. It is
|
||||
recommended to use the largest batch size that fits into TPU memory.
|
||||
|
||||
### Pre-training data
|
||||
|
||||
We will **not** be able to release the pre-processed datasets used in the paper.
|
||||
For Wikipedia, the recommended pre-processing is to download
|
||||
[the latest dump](https://dumps.wikimedia.org/enwiki/latest/enwiki-latest-pages-articles.xml.bz2),
|
||||
extract the text with
|
||||
[`WikiExtractor.py`](https://github.com/attardi/wikiextractor), and then apply
|
||||
any necessary cleanup to convert it into plain text.
|
||||
|
||||
Unfortunately the researchers who collected the
|
||||
[BookCorpus](http://yknzhu.wixsite.com/mbweb) no longer have it available for
|
||||
public download. The
|
||||
[Project Guttenberg Dataset](https://web.eecs.umich.edu/~lahiri/gutenberg_dataset.html)
|
||||
is a somewhat smaller (200M word) collection of older books that are public
|
||||
domain.
|
||||
|
||||
[Common Crawl](http://commoncrawl.org/) is another very large collection of
|
||||
text, but you will likely have to do substantial pre-processing and cleanup to
|
||||
extract a usuable corpus for pre-training BERT.
|
||||
|
||||
### Learning a new WordPiece vocabulary
|
||||
|
||||
This repository does not include code for *learning* a new WordPiece vocabulary.
|
||||
The reason is that the code used in the paper was implemented in C++ with
|
||||
dependencies on Google's internal libraries. For English, it is almost always
|
||||
better to just start with our vocabulary and pre-trained models. For learning
|
||||
vocabularies of other languages, there are a number of open source options
|
||||
available. However, keep in mind that these are not compatible with our
|
||||
`tokenization.py` library:
|
||||
|
||||
* [Google's SentencePiece library](https://github.com/google/sentencepiece)
|
||||
|
||||
* [tensor2tensor's WordPiece generation script](https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/data_generators/text_encoder_build_subword.py)
|
||||
|
||||
* [Rico Sennrich's Byte Pair Encoding library](https://github.com/rsennrich/subword-nmt)
|
||||
|
||||
## Using BERT in Colab
|
||||
|
||||
If you want to use BERT with [Colab](https://colab.sandbox.google.com), you can
|
||||
get started with the notebook
|
||||
"[BERT FineTuning with Cloud TPUs](https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)".
|
||||
**At the time of this writing (October 31st, 2018), Colab users can access a
|
||||
Cloud TPU completely for free.** Note: One per user, availability limited,
|
||||
requires a Google Cloud Platform account with storage (although storage may be
|
||||
purchased with free credit for signing up with GCP), and this capability may not
|
||||
longer be available in the future. Click on the BERT Colab that was just linked
|
||||
for more information.
|
||||
|
||||
## FAQ
|
||||
|
||||
#### Is this code compatible with Cloud TPUs? What about GPUs?
|
||||
|
||||
Yes, all of the code in this repository works out-of-the-box with CPU, GPU, and
|
||||
Cloud TPU. However, GPU training is single-GPU only.
|
||||
|
||||
#### I am getting out-of-memory errors, what is wrong?
|
||||
|
||||
See the section on [out-of-memory issues](#out-of-memory-issues) for more
|
||||
information.
|
||||
|
||||
#### Is there a PyTorch version available?
|
||||
|
||||
There is no official PyTorch implementation. If someone creates a line-for-line
|
||||
PyTorch reimplementation so that our pre-trained checkpoints can be directly
|
||||
converted, we would be happy to link to that PyTorch version here.
|
||||
|
||||
#### Will models in other languages be released?
|
||||
|
||||
Yes, we plan to release a multi-lingual BERT model in the near future. We cannot
|
||||
make promises about exactly which languages will be included, but it will likely
|
||||
be a single model which includes *most* of the languages which have a
|
||||
significantly-sized Wikipedia.
|
||||
|
||||
#### Will models larger than `BERT-Large` be released?
|
||||
|
||||
So far we have not attempted to train anything larger than `BERT-Large`. It is
|
||||
possible that we will release larger models if we are able to obtain significant
|
||||
improvements.
|
||||
|
||||
#### What license is this library released under?
|
||||
|
||||
All code *and* models are released under the Apache 2.0 license. See the
|
||||
`LICENSE` file for more information.
|
||||
|
||||
#### How do I cite BERT?
|
||||
|
||||
For now, cite [the Arxiv paper](https://arxiv.org/abs/1810.04805):
|
||||
|
||||
```
|
||||
@article{devlin2018bert,
|
||||
title={BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding},
|
||||
author={Devlin, Jacob and Chang, Ming-Wei and Lee, Kenton and Toutanova, Kristina},
|
||||
journal={arXiv preprint arXiv:1810.04805},
|
||||
year={2018}
|
||||
}
|
||||
```
|
||||
|
||||
If we submit the paper to a conference or journal, we will update the BibTeX.
|
||||
|
||||
## Disclaimer
|
||||
|
||||
This is not an official Google product.
|
||||
|
||||
## Contact information
|
||||
|
||||
For help or issues using BERT, please submit a GitHub issue.
|
||||
|
||||
For personal communication related to BERT, please contact Jacob Devlin
|
||||
(`jacobdevlin@google.com`), Ming-Wei Chang (`mingweichang@google.com`), or
|
||||
Kenton Lee (`kentonl@google.com`).
|
||||
The main dependencies of this code are:
|
||||
- PyTorch (>= 0.4.0)
|
||||
- tqdm
|
||||
@ -1,429 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Create masked LM/next sentence masked_lm TF examples for BERT."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import collections
|
||||
import random
|
||||
|
||||
import tokenization
|
||||
import tensorflow as tf
|
||||
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
## Required parameters
|
||||
parser.add_argument("--input_file", default=None, type=str, required=True,
|
||||
help="Input raw text file (or comma-separated list of files).")
|
||||
parser.add_argument("--output_file", default=None, type=str, required=True,
|
||||
help="Output TF example file (or comma-separated list of files).")
|
||||
parser.add_argument("--vocab_file", default=None, type=str, required=True,
|
||||
help="The vocabulary file that the BERT model was trained on.")
|
||||
|
||||
## Other parameters
|
||||
parser.add_argument("--do_lower_case", default=True, action='store_true',
|
||||
help="Whether to lower case the input text. Should be True for uncased "
|
||||
"models and False for cased models.")
|
||||
parser.add_argument("--max_seq_length", default=128, type=int, help="Maximum sequence length.")
|
||||
parser.add_argument("--max_predictions_per_seq", default=20, type=int,
|
||||
help="Maximum number of masked LM predictions per sequence.")
|
||||
parser.add_argument("--random_seed", default=12345, type=int, help="Random seed for data generation.")
|
||||
parser.add_argument("--dupe_factor", default=10, type=int,
|
||||
help="Number of times to duplicate the input data (with different masks).")
|
||||
parser.add_argument("--masked_lm_prob", default=0.15, type=float, help="Masked LM probability.")
|
||||
parser.add_argument("--short_seq_prob", default=0.1, type=float,
|
||||
help="Probability of creating sequences which are shorter than the maximum length.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
class TrainingInstance(object):
|
||||
"""A single training instance (sentence pair)."""
|
||||
|
||||
def __init__(self, tokens, segment_ids, masked_lm_positions, masked_lm_labels,
|
||||
is_random_next):
|
||||
self.tokens = tokens
|
||||
self.segment_ids = segment_ids
|
||||
self.is_random_next = is_random_next
|
||||
self.masked_lm_positions = masked_lm_positions
|
||||
self.masked_lm_labels = masked_lm_labels
|
||||
|
||||
def __str__(self):
|
||||
s = ""
|
||||
s += "tokens: %s\n" % (" ".join(
|
||||
[tokenization.printable_text(x) for x in self.tokens]))
|
||||
s += "segment_ids: %s\n" % (" ".join([str(x) for x in self.segment_ids]))
|
||||
s += "is_random_next: %s\n" % self.is_random_next
|
||||
s += "masked_lm_positions: %s\n" % (" ".join(
|
||||
[str(x) for x in self.masked_lm_positions]))
|
||||
s += "masked_lm_labels: %s\n" % (" ".join(
|
||||
[tokenization.printable_text(x) for x in self.masked_lm_labels]))
|
||||
s += "\n"
|
||||
return s
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
def write_instance_to_example_files(instances, tokenizer, max_seq_length,
|
||||
max_predictions_per_seq, output_files):
|
||||
"""Create TF example files from `TrainingInstance`s."""
|
||||
writers = []
|
||||
for output_file in output_files:
|
||||
writers.append(tf.python_io.TFRecordWriter(output_file))
|
||||
|
||||
writer_index = 0
|
||||
|
||||
total_written = 0
|
||||
for (inst_index, instance) in enumerate(instances):
|
||||
input_ids = tokenizer.convert_tokens_to_ids(instance.tokens)
|
||||
input_mask = [1] * len(input_ids)
|
||||
segment_ids = list(instance.segment_ids)
|
||||
assert len(input_ids) <= max_seq_length
|
||||
|
||||
while len(input_ids) < max_seq_length:
|
||||
input_ids.append(0)
|
||||
input_mask.append(0)
|
||||
segment_ids.append(0)
|
||||
|
||||
assert len(input_ids) == max_seq_length
|
||||
assert len(input_mask) == max_seq_length
|
||||
assert len(segment_ids) == max_seq_length
|
||||
|
||||
masked_lm_positions = list(instance.masked_lm_positions)
|
||||
masked_lm_ids = tokenizer.convert_tokens_to_ids(instance.masked_lm_labels)
|
||||
masked_lm_weights = [1.0] * len(masked_lm_ids)
|
||||
|
||||
while len(masked_lm_positions) < max_predictions_per_seq:
|
||||
masked_lm_positions.append(0)
|
||||
masked_lm_ids.append(0)
|
||||
masked_lm_weights.append(0.0)
|
||||
|
||||
next_sentence_label = 1 if instance.is_random_next else 0
|
||||
|
||||
features = collections.OrderedDict()
|
||||
features["input_ids"] = create_int_feature(input_ids)
|
||||
features["input_mask"] = create_int_feature(input_mask)
|
||||
features["segment_ids"] = create_int_feature(segment_ids)
|
||||
features["masked_lm_positions"] = create_int_feature(masked_lm_positions)
|
||||
features["masked_lm_ids"] = create_int_feature(masked_lm_ids)
|
||||
features["masked_lm_weights"] = create_float_feature(masked_lm_weights)
|
||||
features["next_sentence_labels"] = create_int_feature([next_sentence_label])
|
||||
|
||||
tf_example = tf.train.Example(features=tf.train.Features(feature=features))
|
||||
|
||||
writers[writer_index].write(tf_example.SerializeToString())
|
||||
writer_index = (writer_index + 1) % len(writers)
|
||||
|
||||
total_written += 1
|
||||
|
||||
if inst_index < 20:
|
||||
tf.logging.info("*** Example ***")
|
||||
tf.logging.info("tokens: %s" % " ".join(
|
||||
[tokenization.printable_text(x) for x in instance.tokens]))
|
||||
|
||||
for feature_name in features.keys():
|
||||
feature = features[feature_name]
|
||||
values = []
|
||||
if feature.int64_list.value:
|
||||
values = feature.int64_list.value
|
||||
elif feature.float_list.value:
|
||||
values = feature.float_list.value
|
||||
tf.logging.info(
|
||||
"%s: %s" % (feature_name, " ".join([str(x) for x in values])))
|
||||
|
||||
for writer in writers:
|
||||
writer.close()
|
||||
|
||||
tf.logging.info("Wrote %d total instances", total_written)
|
||||
|
||||
|
||||
def create_int_feature(values):
|
||||
feature = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
|
||||
return feature
|
||||
|
||||
|
||||
def create_float_feature(values):
|
||||
feature = tf.train.Feature(float_list=tf.train.FloatList(value=list(values)))
|
||||
return feature
|
||||
|
||||
|
||||
def create_training_instances(input_files, tokenizer, max_seq_length,
|
||||
dupe_factor, short_seq_prob, masked_lm_prob,
|
||||
max_predictions_per_seq, rng):
|
||||
"""Create `TrainingInstance`s from raw text."""
|
||||
all_documents = [[]]
|
||||
|
||||
# Input file format:
|
||||
# (1) One sentence per line. These should ideally be actual sentences, not
|
||||
# entire paragraphs or arbitrary spans of text. (Because we use the
|
||||
# sentence boundaries for the "next sentence prediction" task).
|
||||
# (2) Blank lines between documents. Document boundaries are needed so
|
||||
# that the "next sentence prediction" task doesn't span between documents.
|
||||
for input_file in input_files:
|
||||
with tf.gfile.GFile(input_file, "r") as reader:
|
||||
while True:
|
||||
line = tokenization.convert_to_unicode(reader.readline())
|
||||
if not line:
|
||||
break
|
||||
line = line.strip()
|
||||
|
||||
# Empty lines are used as document delimiters
|
||||
if not line:
|
||||
all_documents.append([])
|
||||
tokens = tokenizer.tokenize(line)
|
||||
if tokens:
|
||||
all_documents[-1].append(tokens)
|
||||
|
||||
# Remove empty documents
|
||||
all_documents = [x for x in all_documents if x]
|
||||
rng.shuffle(all_documents)
|
||||
|
||||
vocab_words = list(tokenizer.vocab.keys())
|
||||
instances = []
|
||||
for _ in range(dupe_factor):
|
||||
for document_index in range(len(all_documents)):
|
||||
instances.extend(
|
||||
create_instances_from_document(
|
||||
all_documents, document_index, max_seq_length, short_seq_prob,
|
||||
masked_lm_prob, max_predictions_per_seq, vocab_words, rng))
|
||||
|
||||
rng.shuffle(instances)
|
||||
return instances
|
||||
|
||||
|
||||
def create_instances_from_document(
|
||||
all_documents, document_index, max_seq_length, short_seq_prob,
|
||||
masked_lm_prob, max_predictions_per_seq, vocab_words, rng):
|
||||
"""Creates `TrainingInstance`s for a single document."""
|
||||
document = all_documents[document_index]
|
||||
|
||||
# Account for [CLS], [SEP], [SEP]
|
||||
max_num_tokens = max_seq_length - 3
|
||||
|
||||
# We *usually* want to fill up the entire sequence since we are padding
|
||||
# to `max_seq_length` anyways, so short sequences are generally wasted
|
||||
# computation. However, we *sometimes*
|
||||
# (i.e., short_seq_prob == 0.1 == 10% of the time) want to use shorter
|
||||
# sequences to minimize the mismatch between pre-training and fine-tuning.
|
||||
# The `target_seq_length` is just a rough target however, whereas
|
||||
# `max_seq_length` is a hard limit.
|
||||
target_seq_length = max_num_tokens
|
||||
if rng.random() < short_seq_prob:
|
||||
target_seq_length = rng.randint(2, max_num_tokens)
|
||||
|
||||
# We DON'T just concatenate all of the tokens from a document into a long
|
||||
# sequence and choose an arbitrary split point because this would make the
|
||||
# next sentence prediction task too easy. Instead, we split the input into
|
||||
# segments "A" and "B" based on the actual "sentences" provided by the user
|
||||
# input.
|
||||
instances = []
|
||||
current_chunk = []
|
||||
current_length = 0
|
||||
i = 0
|
||||
while i < len(document):
|
||||
segment = document[i]
|
||||
current_chunk.append(segment)
|
||||
current_length += len(segment)
|
||||
if i == len(document) - 1 or current_length >= target_seq_length:
|
||||
if current_chunk:
|
||||
# `a_end` is how many segments from `current_chunk` go into the `A`
|
||||
# (first) sentence.
|
||||
a_end = 1
|
||||
if len(current_chunk) >= 2:
|
||||
a_end = rng.randint(1, len(current_chunk) - 1)
|
||||
|
||||
tokens_a = []
|
||||
for j in range(a_end):
|
||||
tokens_a.extend(current_chunk[j])
|
||||
|
||||
tokens_b = []
|
||||
# Random next
|
||||
is_random_next = False
|
||||
if len(current_chunk) == 1 or rng.random() < 0.5:
|
||||
is_random_next = True
|
||||
target_b_length = target_seq_length - len(tokens_a)
|
||||
|
||||
# This should rarely go for more than one iteration for large
|
||||
# corpora. However, just to be careful, we try to make sure that
|
||||
# the random document is not the same as the document
|
||||
# we're processing.
|
||||
for _ in range(10):
|
||||
random_document_index = rng.randint(0, len(all_documents) - 1)
|
||||
if random_document_index != document_index:
|
||||
break
|
||||
|
||||
random_document = all_documents[random_document_index]
|
||||
random_start = rng.randint(0, len(random_document) - 1)
|
||||
for j in range(random_start, len(random_document)):
|
||||
tokens_b.extend(random_document[j])
|
||||
if len(tokens_b) >= target_b_length:
|
||||
break
|
||||
# We didn't actually use these segments so we "put them back" so
|
||||
# they don't go to waste.
|
||||
num_unused_segments = len(current_chunk) - a_end
|
||||
i -= num_unused_segments
|
||||
# Actual next
|
||||
else:
|
||||
is_random_next = False
|
||||
for j in range(a_end, len(current_chunk)):
|
||||
tokens_b.extend(current_chunk[j])
|
||||
truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng)
|
||||
|
||||
assert len(tokens_a) >= 1
|
||||
assert len(tokens_b) >= 1
|
||||
|
||||
tokens = []
|
||||
segment_ids = []
|
||||
tokens.append("[CLS]")
|
||||
segment_ids.append(0)
|
||||
for token in tokens_a:
|
||||
tokens.append(token)
|
||||
segment_ids.append(0)
|
||||
|
||||
tokens.append("[SEP]")
|
||||
segment_ids.append(0)
|
||||
|
||||
for token in tokens_b:
|
||||
tokens.append(token)
|
||||
segment_ids.append(1)
|
||||
tokens.append("[SEP]")
|
||||
segment_ids.append(1)
|
||||
|
||||
(tokens, masked_lm_positions,
|
||||
masked_lm_labels) = create_masked_lm_predictions(
|
||||
tokens, masked_lm_prob, max_predictions_per_seq, vocab_words, rng)
|
||||
instance = TrainingInstance(
|
||||
tokens=tokens,
|
||||
segment_ids=segment_ids,
|
||||
is_random_next=is_random_next,
|
||||
masked_lm_positions=masked_lm_positions,
|
||||
masked_lm_labels=masked_lm_labels)
|
||||
instances.append(instance)
|
||||
current_chunk = []
|
||||
current_length = 0
|
||||
i += 1
|
||||
|
||||
return instances
|
||||
|
||||
|
||||
def create_masked_lm_predictions(tokens, masked_lm_prob,
|
||||
max_predictions_per_seq, vocab_words, rng):
|
||||
"""Creates the predictis for the masked LM objective."""
|
||||
|
||||
cand_indexes = []
|
||||
for (i, token) in enumerate(tokens):
|
||||
if token == "[CLS]" or token == "[SEP]":
|
||||
continue
|
||||
cand_indexes.append(i)
|
||||
|
||||
rng.shuffle(cand_indexes)
|
||||
|
||||
output_tokens = list(tokens)
|
||||
|
||||
masked_lm = collections.namedtuple("masked_lm", ["index", "label"]) # pylint: disable=invalid-name
|
||||
|
||||
num_to_predict = min(max_predictions_per_seq,
|
||||
max(1, int(round(len(tokens) * masked_lm_prob))))
|
||||
|
||||
masked_lms = []
|
||||
covered_indexes = set()
|
||||
for index in cand_indexes:
|
||||
if len(masked_lms) >= num_to_predict:
|
||||
break
|
||||
if index in covered_indexes:
|
||||
continue
|
||||
covered_indexes.add(index)
|
||||
|
||||
masked_token = None
|
||||
# 80% of the time, replace with [MASK]
|
||||
if rng.random() < 0.8:
|
||||
masked_token = "[MASK]"
|
||||
else:
|
||||
# 10% of the time, keep original
|
||||
if rng.random() < 0.5:
|
||||
masked_token = tokens[index]
|
||||
# 10% of the time, replace with random word
|
||||
else:
|
||||
masked_token = vocab_words[rng.randint(0, len(vocab_words) - 1)]
|
||||
|
||||
output_tokens[index] = masked_token
|
||||
|
||||
masked_lms.append(masked_lm(index=index, label=tokens[index]))
|
||||
|
||||
masked_lms = sorted(masked_lms, key=lambda x: x.index)
|
||||
|
||||
masked_lm_positions = []
|
||||
masked_lm_labels = []
|
||||
for p in masked_lms:
|
||||
masked_lm_positions.append(p.index)
|
||||
masked_lm_labels.append(p.label)
|
||||
|
||||
return (output_tokens, masked_lm_positions, masked_lm_labels)
|
||||
|
||||
|
||||
def truncate_seq_pair(tokens_a, tokens_b, max_num_tokens, rng):
|
||||
"""Truncates a pair of sequences to a maximum sequence length."""
|
||||
while True:
|
||||
total_length = len(tokens_a) + len(tokens_b)
|
||||
if total_length <= max_num_tokens:
|
||||
break
|
||||
|
||||
trunc_tokens = tokens_a if len(tokens_a) > len(tokens_b) else tokens_b
|
||||
assert len(trunc_tokens) >= 1
|
||||
|
||||
# We want to sometimes truncate from the front and sometimes from the
|
||||
# back to add more randomness and avoid biases.
|
||||
if rng.random() < 0.5:
|
||||
del trunc_tokens[0]
|
||||
else:
|
||||
trunc_tokens.pop()
|
||||
|
||||
|
||||
def main(_):
|
||||
tf.logging.set_verbosity(tf.logging.INFO)
|
||||
|
||||
tokenizer = tokenization.FullTokenizer(
|
||||
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
|
||||
|
||||
input_files = []
|
||||
for input_pattern in args.input_file.split(","):
|
||||
input_files.extend(tf.gfile.Glob(input_pattern))
|
||||
|
||||
tf.logging.info("*** Reading from input files ***")
|
||||
for input_file in input_files:
|
||||
tf.logging.info(" %s", input_file)
|
||||
|
||||
rng = random.Random(args.random_seed)
|
||||
instances = create_training_instances(
|
||||
input_files, tokenizer, args.max_seq_length, args.dupe_factor,
|
||||
args.short_seq_prob, args.masked_lm_prob, args.max_predictions_per_seq,
|
||||
rng)
|
||||
|
||||
output_files = args.output_file.split(",")
|
||||
tf.logging.info("*** Writing to output files ***")
|
||||
for output_file in output_files:
|
||||
tf.logging.info(" %s", output_file)
|
||||
|
||||
write_instance_to_example_files(instances, tokenizer, args.max_seq_length,
|
||||
args.max_predictions_per_seq, output_files)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tf.app.run()
|
||||
@ -249,6 +249,9 @@ def main():
|
||||
if args.init_checkpoint is not None:
|
||||
model.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
|
||||
model.to(device)
|
||||
|
||||
if n_gpu > 1:
|
||||
model = nn.DataParallel(model)
|
||||
|
||||
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
|
||||
all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)
|
||||
|
||||
@ -502,7 +502,7 @@ class BertForQuestionAnswering(nn.Module):
|
||||
def compute_loss(logits, positions):
|
||||
max_position = positions.max().item()
|
||||
one_hot = torch.FloatTensor(batch_size, max(max_position, seq_length) +1).zero_()
|
||||
one_hot = one_hot.scatter(1, positions.cpu(), 1) # Second argument need to be LongTensor and not cuda.LongTensor
|
||||
one_hot = one_hot.scatter_(1, positions.cpu(), 1) # Second argument need to be LongTensor and not cuda.LongTensor
|
||||
one_hot = one_hot[:, :seq_length].to(input_ids.device)
|
||||
log_probs = nn.functional.log_softmax(logits, dim = -1).view(batch_size, seq_length)
|
||||
loss = -torch.mean(torch.sum(one_hot*log_probs), dim = -1)
|
||||
|
||||
@ -482,6 +482,9 @@ def main():
|
||||
if args.init_checkpoint is not None:
|
||||
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
|
||||
model.to(device)
|
||||
|
||||
if n_gpu > 1:
|
||||
model = torch.nn.DataParallel(model)
|
||||
|
||||
no_decay = ['bias', 'gamma', 'beta']
|
||||
optimizer_parameters = [
|
||||
@ -518,7 +521,7 @@ def main():
|
||||
|
||||
model.train()
|
||||
nb_tr_examples = 0
|
||||
for epoch in trange(args.num_train_epochs, desc="Epoch"):
|
||||
for epoch in trange(int(args.num_train_epochs), desc="Epoch"):
|
||||
for input_ids, input_mask, segment_ids, label_ids in tqdm(train_dataloader, desc="Iteration"):
|
||||
input_ids = input_ids.to(device)
|
||||
input_mask = input_mask.float().to(device)
|
||||
|
||||
@ -1,460 +0,0 @@
|
||||
# coding=utf-8
|
||||
# Copyright 2018 The Google AI Language Team Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Run masked LM/next sentence masked_lm pre-training for BERT."""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import modeling
|
||||
import optimization
|
||||
import tensorflow as tf
|
||||
import argparse
|
||||
|
||||
|
||||
def model_fn_builder(bert_config, init_checkpoint, learning_rate,
|
||||
num_train_steps, num_warmup_steps, use_tpu,
|
||||
use_one_hot_embeddings):
|
||||
"""Returns `model_fn` closure for TPUEstimator."""
|
||||
|
||||
def model_fn(features, labels, mode, params): # pylint: disable=unused-argument
|
||||
"""The `model_fn` for TPUEstimator."""
|
||||
|
||||
tf.logging.info("*** Features ***")
|
||||
for name in sorted(features.keys()):
|
||||
tf.logging.info(" name = %s, shape = %s" % (name, features[name].shape))
|
||||
|
||||
input_ids = features["input_ids"]
|
||||
input_mask = features["input_mask"]
|
||||
segment_ids = features["segment_ids"]
|
||||
masked_lm_positions = features["masked_lm_positions"]
|
||||
masked_lm_ids = features["masked_lm_ids"]
|
||||
masked_lm_weights = features["masked_lm_weights"]
|
||||
next_sentence_labels = features["next_sentence_labels"]
|
||||
|
||||
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
|
||||
|
||||
model = modeling.BertModel(
|
||||
config=bert_config,
|
||||
is_training=is_training,
|
||||
input_ids=input_ids,
|
||||
input_mask=input_mask,
|
||||
token_type_ids=segment_ids,
|
||||
use_one_hot_embeddings=use_one_hot_embeddings)
|
||||
|
||||
(masked_lm_loss,
|
||||
masked_lm_example_loss, masked_lm_log_probs) = get_masked_lm_output(
|
||||
bert_config, model.get_sequence_output(), model.get_embedding_table(),
|
||||
masked_lm_positions, masked_lm_ids, masked_lm_weights)
|
||||
|
||||
(next_sentence_loss, next_sentence_example_loss,
|
||||
next_sentence_log_probs) = get_next_sentence_output(
|
||||
bert_config, model.get_pooled_output(), next_sentence_labels)
|
||||
|
||||
total_loss = masked_lm_loss + next_sentence_loss
|
||||
|
||||
tvars = tf.trainable_variables()
|
||||
|
||||
initialized_variable_names = {}
|
||||
scaffold_fn = None
|
||||
if init_checkpoint:
|
||||
(assignment_map,
|
||||
initialized_variable_names) = modeling.get_assigment_map_from_checkpoint(
|
||||
tvars, init_checkpoint)
|
||||
if use_tpu:
|
||||
|
||||
def tpu_scaffold():
|
||||
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
|
||||
return tf.train.Scaffold()
|
||||
|
||||
scaffold_fn = tpu_scaffold
|
||||
else:
|
||||
tf.train.init_from_checkpoint(init_checkpoint, assignment_map)
|
||||
|
||||
tf.logging.info("**** Trainable Variables ****")
|
||||
for var in tvars:
|
||||
init_string = ""
|
||||
if var.name in initialized_variable_names:
|
||||
init_string = ", *INIT_FROM_CKPT*"
|
||||
tf.logging.info(" name = %s, shape = %s%s", var.name, var.shape,
|
||||
init_string)
|
||||
|
||||
output_spec = None
|
||||
if mode == tf.estimator.ModeKeys.TRAIN:
|
||||
train_op = optimization.create_optimizer(
|
||||
total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)
|
||||
|
||||
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
|
||||
mode=mode,
|
||||
loss=total_loss,
|
||||
train_op=train_op,
|
||||
scaffold_fn=scaffold_fn)
|
||||
elif mode == tf.estimator.ModeKeys.EVAL:
|
||||
|
||||
def metric_fn(masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids,
|
||||
masked_lm_weights, next_sentence_example_loss,
|
||||
next_sentence_log_probs, next_sentence_labels):
|
||||
"""Computes the loss and accuracy of the model."""
|
||||
masked_lm_log_probs = tf.reshape(masked_lm_log_probs,
|
||||
[-1, masked_lm_log_probs.shape[-1]])
|
||||
masked_lm_predictions = tf.argmax(
|
||||
masked_lm_log_probs, axis=-1, output_type=tf.int32)
|
||||
masked_lm_example_loss = tf.reshape(masked_lm_example_loss, [-1])
|
||||
masked_lm_ids = tf.reshape(masked_lm_ids, [-1])
|
||||
masked_lm_weights = tf.reshape(masked_lm_weights, [-1])
|
||||
masked_lm_accuracy = tf.metrics.accuracy(
|
||||
labels=masked_lm_ids,
|
||||
predictions=masked_lm_predictions,
|
||||
weights=masked_lm_weights)
|
||||
masked_lm_mean_loss = tf.metrics.mean(
|
||||
values=masked_lm_example_loss, weights=masked_lm_weights)
|
||||
|
||||
next_sentence_log_probs = tf.reshape(
|
||||
next_sentence_log_probs, [-1, next_sentence_log_probs.shape[-1]])
|
||||
next_sentence_predictions = tf.argmax(
|
||||
next_sentence_log_probs, axis=-1, output_type=tf.int32)
|
||||
next_sentence_labels = tf.reshape(next_sentence_labels, [-1])
|
||||
next_sentence_accuracy = tf.metrics.accuracy(
|
||||
labels=next_sentence_labels, predictions=next_sentence_predictions)
|
||||
next_sentence_mean_loss = tf.metrics.mean(
|
||||
values=next_sentence_example_loss)
|
||||
|
||||
return {
|
||||
"masked_lm_accuracy": masked_lm_accuracy,
|
||||
"masked_lm_loss": masked_lm_mean_loss,
|
||||
"next_sentence_accuracy": next_sentence_accuracy,
|
||||
"next_sentence_loss": next_sentence_mean_loss,
|
||||
}
|
||||
|
||||
eval_metrics = (metric_fn, [
|
||||
masked_lm_example_loss, masked_lm_log_probs, masked_lm_ids,
|
||||
masked_lm_weights, next_sentence_example_loss,
|
||||
next_sentence_log_probs, next_sentence_labels
|
||||
])
|
||||
output_spec = tf.contrib.tpu.TPUEstimatorSpec(
|
||||
mode=mode,
|
||||
loss=total_loss,
|
||||
eval_metrics=eval_metrics,
|
||||
scaffold_fn=scaffold_fn)
|
||||
else:
|
||||
raise ValueError("Only TRAIN and EVAL modes are supported: %s" % (mode))
|
||||
|
||||
return output_spec
|
||||
|
||||
return model_fn
|
||||
|
||||
|
||||
def get_masked_lm_output(bert_config, input_tensor, output_weights, positions,
|
||||
label_ids, label_weights):
|
||||
"""Get loss and log probs for the masked LM."""
|
||||
input_tensor = gather_indexes(input_tensor, positions)
|
||||
|
||||
with tf.variable_scope("cls/predictions"):
|
||||
# We apply one more non-linear transformation before the output layer.
|
||||
# This matrix is not used after pre-training.
|
||||
with tf.variable_scope("transform"):
|
||||
input_tensor = tf.layers.dense(
|
||||
input_tensor,
|
||||
units=bert_config.hidden_size,
|
||||
activation=modeling.get_activation(bert_config.hidden_act),
|
||||
kernel_initializer=modeling.create_initializer(
|
||||
bert_config.initializer_range))
|
||||
input_tensor = modeling.layer_norm(input_tensor)
|
||||
|
||||
# The output weights are the same as the input embeddings, but there is
|
||||
# an output-only bias for each token.
|
||||
output_bias = tf.get_variable(
|
||||
"output_bias",
|
||||
shape=[bert_config.vocab_size],
|
||||
initializer=tf.zeros_initializer())
|
||||
logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
|
||||
logits = tf.nn.bias_add(logits, output_bias)
|
||||
log_probs = tf.nn.log_softmax(logits, axis=-1)
|
||||
|
||||
label_ids = tf.reshape(label_ids, [-1])
|
||||
label_weights = tf.reshape(label_weights, [-1])
|
||||
|
||||
one_hot_labels = tf.one_hot(
|
||||
label_ids, depth=bert_config.vocab_size, dtype=tf.float32)
|
||||
|
||||
# The `positions` tensor might be zero-padded (if the sequence is too
|
||||
# short to have the maximum number of predictions). The `label_weights`
|
||||
# tensor has a value of 1.0 for every real prediction and 0.0 for the
|
||||
# padding predictions.
|
||||
per_example_loss = -tf.reduce_sum(log_probs * one_hot_labels, axis=[-1])
|
||||
numerator = tf.reduce_sum(label_weights * per_example_loss)
|
||||
denominator = tf.reduce_sum(label_weights) + 1e-5
|
||||
loss = numerator / denominator
|
||||
|
||||
return (loss, per_example_loss, log_probs)
|
||||
|
||||
|
||||
def get_next_sentence_output(bert_config, input_tensor, labels):
|
||||
"""Get loss and log probs for the next sentence prediction."""
|
||||
|
||||
# Simple binary classification. Note that 0 is "next sentence" and 1 is
|
||||
# "random sentence". This weight matrix is not used after pre-training.
|
||||
with tf.variable_scope("cls/seq_relationship"):
|
||||
output_weights = tf.get_variable(
|
||||
"output_weights",
|
||||
shape=[2, bert_config.hidden_size],
|
||||
initializer=modeling.create_initializer(bert_config.initializer_range))
|
||||
output_bias = tf.get_variable(
|
||||
"output_bias", shape=[2], initializer=tf.zeros_initializer())
|
||||
|
||||
logits = tf.matmul(input_tensor, output_weights, transpose_b=True)
|
||||
logits = tf.nn.bias_add(logits, output_bias)
|
||||
log_probs = tf.nn.log_softmax(logits, axis=-1)
|
||||
labels = tf.reshape(labels, [-1])
|
||||
one_hot_labels = tf.one_hot(labels, depth=2, dtype=tf.float32)
|
||||
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)
|
||||
loss = tf.reduce_mean(per_example_loss)
|
||||
return (loss, per_example_loss, log_probs)
|
||||
|
||||
|
||||
def gather_indexes(sequence_tensor, positions):
|
||||
"""Gathers the vectors at the specific positions over a minibatch."""
|
||||
sequence_shape = modeling.get_shape_list(sequence_tensor, expected_rank=3)
|
||||
batch_size = sequence_shape[0]
|
||||
seq_length = sequence_shape[1]
|
||||
width = sequence_shape[2]
|
||||
|
||||
flat_offsets = tf.reshape(
|
||||
tf.range(0, batch_size, dtype=tf.int32) * seq_length, [-1, 1])
|
||||
flat_positions = tf.reshape(positions + flat_offsets, [-1])
|
||||
flat_sequence_tensor = tf.reshape(sequence_tensor,
|
||||
[batch_size * seq_length, width])
|
||||
output_tensor = tf.gather(flat_sequence_tensor, flat_positions)
|
||||
return output_tensor
|
||||
|
||||
|
||||
def input_fn_builder(input_files,
|
||||
max_seq_length,
|
||||
max_predictions_per_seq,
|
||||
is_training,
|
||||
num_cpu_threads=4):
|
||||
"""Creates an `input_fn` closure to be passed to TPUEstimator."""
|
||||
|
||||
def input_fn(params):
|
||||
"""The actual input function."""
|
||||
batch_size = params["batch_size"]
|
||||
|
||||
name_to_features = {
|
||||
"input_ids":
|
||||
tf.FixedLenFeature([max_seq_length], tf.int64),
|
||||
"input_mask":
|
||||
tf.FixedLenFeature([max_seq_length], tf.int64),
|
||||
"segment_ids":
|
||||
tf.FixedLenFeature([max_seq_length], tf.int64),
|
||||
"masked_lm_positions":
|
||||
tf.FixedLenFeature([max_predictions_per_seq], tf.int64),
|
||||
"masked_lm_ids":
|
||||
tf.FixedLenFeature([max_predictions_per_seq], tf.int64),
|
||||
"masked_lm_weights":
|
||||
tf.FixedLenFeature([max_predictions_per_seq], tf.float32),
|
||||
"next_sentence_labels":
|
||||
tf.FixedLenFeature([1], tf.int64),
|
||||
}
|
||||
|
||||
# For training, we want a lot of parallel reading and shuffling.
|
||||
# For eval, we want no shuffling and parallel reading doesn't matter.
|
||||
if is_training:
|
||||
d = tf.data.Dataset.from_tensor_slices(tf.constant(input_files))
|
||||
d = d.repeat()
|
||||
d = d.shuffle(buffer_size=len(input_files))
|
||||
|
||||
# `cycle_length` is the number of parallel files that get read.
|
||||
cycle_length = min(num_cpu_threads, len(input_files))
|
||||
|
||||
# `sloppy` mode means that the interleaving is not exact. This adds
|
||||
# even more randomness to the training pipeline.
|
||||
d = d.apply(
|
||||
tf.contrib.data.parallel_interleave(
|
||||
tf.data.TFRecordDataset,
|
||||
sloppy=is_training,
|
||||
cycle_length=cycle_length))
|
||||
d = d.shuffle(buffer_size=100)
|
||||
else:
|
||||
d = tf.data.TFRecordDataset(input_files)
|
||||
# Since we evaluate for a fixed number of steps we don't want to encounter
|
||||
# out-of-range exceptions.
|
||||
d = d.repeat()
|
||||
|
||||
# We must `drop_remainder` on training because the TPU requires fixed
|
||||
# size dimensions. For eval, we assume we are evaling on the CPU or GPU
|
||||
# and we *don"t* want to drop the remainder, otherwise we wont cover
|
||||
# every sample.
|
||||
d = d.apply(
|
||||
tf.contrib.data.map_and_batch(
|
||||
lambda record: _decode_record(record, name_to_features),
|
||||
batch_size=batch_size,
|
||||
num_parallel_batches=num_cpu_threads,
|
||||
drop_remainder=True))
|
||||
return d
|
||||
|
||||
return input_fn
|
||||
|
||||
|
||||
def _decode_record(record, name_to_features):
|
||||
"""Decodes a record to a TensorFlow example."""
|
||||
example = tf.parse_single_example(record, name_to_features)
|
||||
|
||||
# tf.Example only supports tf.int64, but the TPU only supports tf.int32.
|
||||
# So cast all int64 to int32.
|
||||
for name in list(example.keys()):
|
||||
t = example[name]
|
||||
if t.dtype == tf.int64:
|
||||
t = tf.to_int32(t)
|
||||
example[name] = t
|
||||
|
||||
return example
|
||||
|
||||
|
||||
def main(_):
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
## Required parameters
|
||||
parser.add_argument("--bert_config_file", default=None, type=str, required=True,
|
||||
help="The config json file corresponding to the pre-trained BERT model. "
|
||||
"This specifies the model architecture.")
|
||||
parser.add_argument("--input_file", default=None, type=str, required=True,
|
||||
help="Input TF example files (can be a glob or comma separated).")
|
||||
parser.add_argument("--output_dir", default=None, type=str, required=True,
|
||||
help="The output directory where the model checkpoints will be written.")
|
||||
|
||||
## Other parameters
|
||||
parser.add_argument("--init_checkpoint", default=None, type=str,
|
||||
help="Initial checkpoint (usually from a pre-trained BERT model).")
|
||||
parser.add_argument("--max_seq_length", default=128, type=int,
|
||||
help="The maximum total input sequence length after WordPiece tokenization. Sequences longer "
|
||||
"than this will be truncated, and sequences shorter than this will be padded. "
|
||||
"Must match data generation.")
|
||||
parser.add_argument("--max_predictions_per_seq", default=20, type=int,
|
||||
help="Maximum number of masked LM predictions per sequence. Must match data generation.")
|
||||
parser.add_argument("--do_train", default=False, action='store_true', help="Whether to run training.")
|
||||
parser.add_argument("--do_eval", default=False, action='store_true', help="Whether to run eval on the dev set.")
|
||||
parser.add_argument("--train_batch_size", default=32, type=int, help="Total batch size for training.")
|
||||
parser.add_argument("--eval_batch_size", default=8, type=int, help="Total batch size for eval.")
|
||||
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
|
||||
parser.add_argument("--num_train_steps", default=100000, type=int, help="Number of training steps.")
|
||||
parser.add_argument("--num_warmup_steps", default=10000, type=int, help="Number of warmup steps.")
|
||||
parser.add_argument("--save_checkpoints_steps", default=1000, type=int,
|
||||
help="How often to save the model checkpoint.")
|
||||
parser.add_argument("--iterations_per_loop", default=1000, type=int,
|
||||
help="How many steps to make in each estimator call.")
|
||||
parser.add_argument("--max_eval_steps", default=100, type=int, help="Maximum number of eval steps.")
|
||||
### BEGIN - TO DELETE EVENTUALLY --> NO SENSE IN PYTORCH ###
|
||||
parser.add_argument("--use_tpu", default=False, action='store_true', help="Whether to use TPU or GPU/CPU.")
|
||||
parser.add_argument("--tpu_name", default=None, type=str,
|
||||
help="The Cloud TPU to use for training. This should be either the name used when creating the "
|
||||
"Cloud TPU, or a grpc://ip.address.of.tpu:8470 url.")
|
||||
parser.add_argument("--tpu_zone", default=None, type=str,
|
||||
help="[Optional] GCE zone where the Cloud TPU is located in. If not specified, we will attempt "
|
||||
"to automatically detect the GCE project from metadata.")
|
||||
parser.add_argument("--gcp_project", default=None, type=str,
|
||||
help="[Optional] Project name for the Cloud TPU-enabled project. If not specified, "
|
||||
"we will attempt to automatically detect the GCE project from metadata.")
|
||||
parser.add_argument("--master", default=None, type=str, help="[Optional] TensorFlow master URL.")
|
||||
parser.add_argument("--num_tpu_cores", default=8, type=int,
|
||||
help="Only used if `use_tpu` is True. Total number of TPU cores to use.")
|
||||
### END - TO DELETE EVENTUALLY --> NO SENSE IN PYTORCH ###
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
tf.logging.set_verbosity(tf.logging.INFO)
|
||||
|
||||
if not args.do_train and not args.do_eval:
|
||||
raise ValueError("At least one of `do_train` or `do_eval` must be True.")
|
||||
|
||||
bert_config = modeling.BertConfig.from_json_file(args.bert_config_file)
|
||||
|
||||
tf.gfile.MakeDirs(args.output_dir)
|
||||
|
||||
input_files = []
|
||||
for input_pattern in args.input_file.split(","):
|
||||
input_files.extend(tf.gfile.Glob(input_pattern))
|
||||
|
||||
tf.logging.info("*** Input Files ***")
|
||||
for input_file in input_files:
|
||||
tf.logging.info(" %s" % input_file)
|
||||
|
||||
tpu_cluster_resolver = None
|
||||
if args.use_tpu and args.tpu_name:
|
||||
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
|
||||
args.tpu_name, zone=args.tpu_zone, project=args.gcp_project)
|
||||
|
||||
is_per_host = tf.contrib.tpu.InputPipelineConfig.PER_HOST_V2
|
||||
run_config = tf.contrib.tpu.RunConfig(
|
||||
cluster=tpu_cluster_resolver,
|
||||
master=args.master,
|
||||
model_dir=args.output_dir,
|
||||
save_checkpoints_steps=args.save_checkpoints_steps,
|
||||
tpu_config=tf.contrib.tpu.TPUConfig(
|
||||
iterations_per_loop=args.iterations_per_loop,
|
||||
num_shards=args.num_tpu_cores,
|
||||
per_host_input_for_training=is_per_host))
|
||||
|
||||
model_fn = model_fn_builder(
|
||||
bert_config=bert_config,
|
||||
init_checkpoint=args.init_checkpoint,
|
||||
learning_rate=args.learning_rate,
|
||||
num_train_steps=args.num_train_steps,
|
||||
num_warmup_steps=args.num_warmup_steps,
|
||||
use_tpu=args.use_tpu,
|
||||
use_one_hot_embeddings=args.use_tpu)
|
||||
|
||||
# If TPU is not available, this will fall back to normal Estimator on CPU
|
||||
# or GPU.
|
||||
estimator = tf.contrib.tpu.TPUEstimator(
|
||||
use_tpu=args.use_tpu,
|
||||
model_fn=model_fn,
|
||||
config=run_config,
|
||||
train_batch_size=args.train_batch_size,
|
||||
eval_batch_size=args.eval_batch_size)
|
||||
|
||||
if args.do_train:
|
||||
tf.logging.info("***** Running training *****")
|
||||
tf.logging.info(" Batch size = %d", args.train_batch_size)
|
||||
train_input_fn = input_fn_builder(
|
||||
input_files=input_files,
|
||||
max_seq_length=args.max_seq_length,
|
||||
max_predictions_per_seq=args.max_predictions_per_seq,
|
||||
is_training=True)
|
||||
estimator.train(input_fn=train_input_fn, max_steps=args.num_train_steps)
|
||||
|
||||
if args.do_eval:
|
||||
tf.logging.info("***** Running evaluation *****")
|
||||
tf.logging.info(" Batch size = %d", args.eval_batch_size)
|
||||
|
||||
eval_input_fn = input_fn_builder(
|
||||
input_files=input_files,
|
||||
max_seq_length=args.max_seq_length,
|
||||
max_predictions_per_seq=args.max_predictions_per_seq,
|
||||
is_training=False)
|
||||
|
||||
result = estimator.evaluate(
|
||||
input_fn=eval_input_fn, steps=args.max_eval_steps)
|
||||
|
||||
output_eval_file = os.path.join(args.output_dir, "eval_results.txt")
|
||||
with tf.gfile.GFile(output_eval_file, "w") as writer:
|
||||
tf.logging.info("***** Eval results *****")
|
||||
for key in sorted(result.keys()):
|
||||
tf.logging.info(" %s = %s", key, str(result[key]))
|
||||
writer.write("%s = %s\n" % (key, str(result[key])))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tf.app.run()
|
||||
@ -23,7 +23,7 @@ import logging
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import tokenization
|
||||
import tokenization_pytorch
|
||||
import six
|
||||
import argparse
|
||||
|
||||
@ -62,9 +62,9 @@ class SquadExample(object):
|
||||
|
||||
def __repr__(self):
|
||||
s = ""
|
||||
s += "qas_id: %s" % (tokenization.printable_text(self.qas_id))
|
||||
s += "qas_id: %s" % (tokenization_pytorch.printable_text(self.qas_id))
|
||||
s += ", question_text: %s" % (
|
||||
tokenization.printable_text(self.question_text))
|
||||
tokenization_pytorch.printable_text(self.question_text))
|
||||
s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens))
|
||||
if self.start_position:
|
||||
s += ", start_position: %d" % (self.start_position)
|
||||
@ -153,7 +153,7 @@ def read_squad_examples(input_file, is_training):
|
||||
# guaranteed to be preserved.
|
||||
actual_text = " ".join(doc_tokens[start_position:(end_position + 1)])
|
||||
cleaned_answer_text = " ".join(
|
||||
tokenization.whitespace_tokenize(orig_answer_text))
|
||||
tokenization_pytorch.whitespace_tokenize(orig_answer_text))
|
||||
if actual_text.find(cleaned_answer_text) == -1:
|
||||
logger.warning("Could not find answer: '%s' vs. '%s'",
|
||||
actual_text, cleaned_answer_text)
|
||||
@ -287,7 +287,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
|
||||
logger.info("example_index: %s" % (example_index))
|
||||
logger.info("doc_span_index: %s" % (doc_span_index))
|
||||
logger.info("tokens: %s" % " ".join(
|
||||
[tokenization.printable_text(x) for x in tokens]))
|
||||
[tokenization_pytorch.printable_text(x) for x in tokens]))
|
||||
logger.info("token_to_orig_map: %s" % " ".join(
|
||||
["%d:%d" % (x, y) for (x, y) in six.iteritems(token_to_orig_map)]))
|
||||
logger.info("token_is_max_context: %s" % " ".join([
|
||||
@ -303,7 +303,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length,
|
||||
logger.info("start_position: %d" % (start_position))
|
||||
logger.info("end_position: %d" % (end_position))
|
||||
logger.info(
|
||||
"answer: %s" % (tokenization.printable_text(answer_text)))
|
||||
"answer: %s" % (tokenization_pytorch.printable_text(answer_text)))
|
||||
|
||||
features.append(
|
||||
InputFeatures(
|
||||
@ -579,7 +579,7 @@ def get_final_text(pred_text, orig_text, do_lower_case):
|
||||
# and `pred_text`, and check if they are the same length. If they are
|
||||
# NOT the same length, the heuristic has failed. If they are the same
|
||||
# length, we assume the characters are one-to-one aligned.
|
||||
tokenizer = tokenization.BasicTokenizer(do_lower_case=do_lower_case)
|
||||
tokenizer = tokenization_pytorch.BasicTokenizer(do_lower_case=do_lower_case)
|
||||
|
||||
tok_text = " ".join(tokenizer.tokenize(orig_text))
|
||||
|
||||
@ -780,7 +780,7 @@ def main():
|
||||
raise ValueError("Output directory () already exists and is not empty.")
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
|
||||
tokenizer = tokenization.FullTokenizer(
|
||||
tokenizer = tokenization_pytorch.FullTokenizer(
|
||||
vocab_file=args.vocab_file, do_lower_case=args.do_lower_case)
|
||||
|
||||
train_examples = None
|
||||
@ -795,6 +795,9 @@ def main():
|
||||
if args.init_checkpoint is not None:
|
||||
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
|
||||
model.to(device)
|
||||
|
||||
if n_gpu > 1:
|
||||
model = torch.nn.DataParallel(model)
|
||||
|
||||
optimizer = BERTAdam([{'params': [p for n, p in model.named_parameters() if n != 'bias'], 'l2': 0.01},
|
||||
{'params': [p for n, p in model.named_parameters() if n == 'bias'], 'l2': 0.}
|
||||
|
||||
@ -21,7 +21,7 @@ from __future__ import print_function
|
||||
import collections
|
||||
import random
|
||||
|
||||
import tokenization
|
||||
from tensorflow_code import tokenization
|
||||
import tensorflow as tf
|
||||
|
||||
flags = tf.flags
|
||||
|
||||
@ -23,8 +23,8 @@ import collections
|
||||
import json
|
||||
import re
|
||||
|
||||
import modeling
|
||||
import tokenization
|
||||
from tensorflow_code import modeling
|
||||
from tensorflow_code import tokenization
|
||||
import tensorflow as tf
|
||||
|
||||
flags = tf.flags
|
||||
|
||||
@ -21,7 +21,7 @@ import json
|
||||
import random
|
||||
import re
|
||||
|
||||
import modeling
|
||||
from tensorflow_code import modeling
|
||||
import six
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
@ -16,7 +16,7 @@ from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import optimization
|
||||
from tensorflow_code import optimization
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
|
||||
@ -20,9 +20,9 @@ from __future__ import print_function
|
||||
|
||||
import csv
|
||||
import os
|
||||
import modeling
|
||||
import optimization
|
||||
import tokenization
|
||||
from tensorflow_code import modeling
|
||||
from tensorflow_code import optimization
|
||||
from tensorflow_code import tokenization
|
||||
import tensorflow as tf
|
||||
|
||||
flags = tf.flags
|
||||
|
||||
@ -19,8 +19,8 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import os
|
||||
import modeling
|
||||
import optimization
|
||||
from tensorflow_code import modeling
|
||||
from tensorflow_code import optimization
|
||||
import tensorflow as tf
|
||||
|
||||
flags = tf.flags
|
||||
|
||||
@ -22,9 +22,9 @@ import collections
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import modeling
|
||||
import optimization
|
||||
import tokenization
|
||||
from tensorflow_code import modeling
|
||||
from tensorflow_code import optimization
|
||||
from tensorflow_code import tokenization
|
||||
import six
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
@ -19,7 +19,7 @@ from __future__ import print_function
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import tokenization
|
||||
from tensorflow_code import tokenization
|
||||
import tensorflow as tf
|
||||
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user