riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 18:22:34 +06:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

quick fix on concatenating text to support more datasets (#8474)

Browse Source
This commit is contained in:
zeyuyun1 2020-11-12 06:47:08 -08:00 committed by GitHub
parent 17b1fd804f
commit 924c624a46
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/language-modeling/run_clm.py
View File

@ -254,7 +254,7 @@ def main():
tokenize_function,
batched=True,
num_proc=data_args.preprocessing_num_workers,
remove_columns=[text_column_name],
remove_columns=column_names,
load_from_cache_file=not data_args.overwrite_cache,
)

2
examples/language-modeling/run_mlm.py
View File

@ -292,7 +292,7 @@ def main():
tokenize_function,
batched=True,
num_proc=data_args.preprocessing_num_workers,
remove_columns=[text_column_name],
remove_columns=column_names,
load_from_cache_file=not data_args.overwrite_cache,
)

2
examples/language-modeling/run_plm.py
View File

@ -279,7 +279,7 @@ def main():
tokenize_function,
batched=True,
num_proc=data_args.preprocessing_num_workers,
remove_columns=[text_column_name],
remove_columns=column_names,
load_from_cache_file=not data_args.overwrite_cache,
)