riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-08-02 19:21:31 +06:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

fix ram efficient fsdp init (#25686)

Browse Source
This commit is contained in:
Sourab Mangrulkar 2023-08-24 11:30:42 +05:30 committed by GitHub
parent 68fa9a5937
commit b85b88069a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/transformers/modeling_utils.py
View File

@ -113,11 +113,11 @@ _init_weights = True
def is_fsdp_enabled():
return strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
return torch.distributed.is_initialized() and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
def is_fsdp_enabled_and_dist_rank_0():
return is_fsdp_enabled() and torch.distributed.is_initialized() and torch.distributed.get_rank() == 0
return is_fsdp_enabled() and torch.distributed.get_rank() == 0
if is_sagemaker_mp_enabled():