From bdfefbadaf7553d8a104dce096fd2905692f8197 Mon Sep 17 00:00:00 2001
From: Sourab Mangrulkar <13534540+pacman100@users.noreply.github.com>
Date: Wed, 15 May 2024 19:32:31 +0530
Subject: [PATCH] Update ds_config_zero3.json (#30829)

---
 tests/deepspeed/ds_config_zero3.json | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/tests/deepspeed/ds_config_zero3.json b/tests/deepspeed/ds_config_zero3.json
index 4d7a154c9b0..dd741ac8473 100644
--- a/tests/deepspeed/ds_config_zero3.json
+++ b/tests/deepspeed/ds_config_zero3.json
@@ -7,11 +7,9 @@
         "hysteresis": 2,
         "min_loss_scale": 1
     },
-
     "bf16": {
         "enabled": "auto"
     },
-
     "optimizer": {
         "type": "AdamW",
         "params": {
@@ -21,7 +19,6 @@
             "weight_decay": "auto"
         }
     },
-
     "scheduler": {
         "type": "WarmupLR",
         "params": {
@@ -30,15 +27,14 @@
             "warmup_num_steps": "auto"
         }
     },
-
     "zero_optimization": {
         "stage": 3,
         "offload_optimizer": {
-            "device": "cpu",
+            "device": "none",
             "pin_memory": true
         },
         "offload_param": {
-            "device": "cpu",
+            "device": "none",
             "pin_memory": true
         },
         "overlap_comm": true,
@@ -51,11 +47,10 @@
         "stage3_max_reuse_distance": 1e9,
         "stage3_gather_16bit_weights_on_model_save": true
     },
-
     "gradient_accumulation_steps": "auto",
     "gradient_clipping": "auto",
     "steps_per_print": 2000,
     "train_batch_size": "auto",
     "train_micro_batch_size_per_gpu": "auto",
     "wall_clock_breakdown": false
-}
+}
\ No newline at end of file