riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-14 10:08:29 +06:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

fixing learning rate schedule when using gradient_accumulation_steps

Browse Source
This commit is contained in:
thomwolf 2018-11-10 16:11:14 +01:00
parent ea85cca8ab
commit a81a1ef8e9
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
run_classifier.py
View File

@ -464,7 +464,7 @@ def main():
if args.do_train: if args.do_train:
train_examples = processor.get_train_examples(args.data_dir) train_examples = processor.get_train_examples(args.data_dir)
num_train_steps = int( num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs) len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
model = BertForSequenceClassification(bert_config, len(label_list)) model = BertForSequenceClassification(bert_config, len(label_list))
if args.init_checkpoint is not None: if args.init_checkpoint is not None:

18
run_squad.py
View File

@ -742,6 +742,10 @@ def main():
default=False, default=False,
action='store_true', action='store_true',
help="Whether to perform optimization and keep the optimizer averages on CPU") help="Whether to perform optimization and keep the optimizer averages on CPU")
parser.add_argument('--fp16',
default=False,
action='store_true',
help="Whether to use 16-bit float precision instead of 32-bit")
args = parser.parse_args() args = parser.parse_args()
@ -801,11 +805,13 @@ def main():
train_examples = read_squad_examples( train_examples = read_squad_examples(
input_file=args.train_file, is_training=True) input_file=args.train_file, is_training=True)
num_train_steps = int( num_train_steps = int(
len(train_examples) / args.train_batch_size * args.num_train_epochs) len(train_examples) / args.train_batch_size / args.gradient_accumulation_steps * args.num_train_epochs)
model = BertForQuestionAnswering(bert_config) model = BertForQuestionAnswering(bert_config)
if args.init_checkpoint is not None: if args.init_checkpoint is not None:
model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu')) model.bert.load_state_dict(torch.load(args.init_checkpoint, map_location='cpu'))
if args.fp16:
model.half()
if not args.optimize_on_cpu: if not args.optimize_on_cpu:
model.to(device) model.to(device)
@ -847,6 +853,12 @@ def main():
all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long) all_start_positions = torch.tensor([f.start_position for f in train_features], dtype=torch.long)
all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long) all_end_positions = torch.tensor([f.end_position for f in train_features], dtype=torch.long)
if args.fp16:
(all_input_ids, all_input_mask,
all_segment_ids, all_start_positions,
all_end_positions) = tuple(t.half() for t in (all_input_ids, all_input_mask, all_segment_ids,
all_start_positions, all_end_positions))
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
all_start_positions, all_end_positions) all_start_positions, all_end_positions)
if args.local_rank == -1: if args.local_rank == -1:
@ -895,6 +907,10 @@ def main():
all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long) all_input_mask = torch.tensor([f.input_mask for f in eval_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long) all_segment_ids = torch.tensor([f.segment_ids for f in eval_features], dtype=torch.long)
all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long)
if args.fp16:
(all_input_ids, all_input_mask,
all_segment_ids, all_example_index) = tuple(t.half() for t in (all_input_ids, all_input_mask,
all_segment_ids, all_example_index))
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_example_index) eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_example_index)
if args.local_rank == -1: if args.local_rank == -1: