riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-03 12:50:06 +06:00
Code Issues Actions 28 Packages Projects Releases Wiki Activity

Update ds_config_zero3.json (#30829)

Browse Source
This commit is contained in:
Sourab Mangrulkar 2024-05-15 19:32:31 +05:30 committed by GitHub
parent 92544cb8f3
commit bdfefbadaf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tests/deepspeed/ds_config_zero3.json
View File

@ -7,11 +7,9 @@
"hysteresis": 2, "hysteresis": 2,
"min_loss_scale": 1 "min_loss_scale": 1
}, },
"bf16": { "bf16": {
"enabled": "auto" "enabled": "auto"
}, },
"optimizer": { "optimizer": {
"type": "AdamW", "type": "AdamW",
"params": { "params": {
@ -21,7 +19,6 @@
"weight_decay": "auto" "weight_decay": "auto"
} }
}, },
"scheduler": { "scheduler": {
"type": "WarmupLR", "type": "WarmupLR",
"params": { "params": {
@ -30,15 +27,14 @@
"warmup_num_steps": "auto" "warmup_num_steps": "auto"
} }
}, },
"zero_optimization": { "zero_optimization": {
"stage": 3, "stage": 3,
"offload_optimizer": { "offload_optimizer": {
"device": "cpu", "device": "none",
"pin_memory": true "pin_memory": true
}, },
"offload_param": { "offload_param": {
"device": "cpu", "device": "none",
"pin_memory": true "pin_memory": true
}, },
"overlap_comm": true, "overlap_comm": true,
@ -51,11 +47,10 @@
"stage3_max_reuse_distance": 1e9, "stage3_max_reuse_distance": 1e9,
"stage3_gather_16bit_weights_on_model_save": true "stage3_gather_16bit_weights_on_model_save": true
}, },
"gradient_accumulation_steps": "auto", "gradient_accumulation_steps": "auto",
"gradient_clipping": "auto", "gradient_clipping": "auto",
"steps_per_print": 2000, "steps_per_print": 2000,
"train_batch_size": "auto", "train_batch_size": "auto",
"train_micro_batch_size_per_gpu": "auto", "train_micro_batch_size_per_gpu": "auto",
"wall_clock_breakdown": false "wall_clock_breakdown": false
} }