riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-31 02:02:21 +06:00
Code Issues Actions 33 Packages Projects Releases Wiki Activity

add gpt2 test on XPU (#37028)

Browse Source 
* add gpt2 test on XPU

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* auto dtype has been fixed

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

* convert model to train mode

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>

---------

Signed-off-by: jiqing-feng <jiqing.feng@intel.com>
This commit is contained in:
jiqing-feng 2025-04-01 17:09:29 +08:00 committed by GitHub
parent 4b13a02920
commit 3a6ab46a0b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 2 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/quantization/bnb/test_4bit.py
View File

@ -626,7 +626,6 @@ class Bnb4BitTestTraining(Base4bitTest):
@apply_skip_if_not_implemented
@unittest.skipIf(torch_device == "xpu", reason="XPU has precision issue on gpt model, will test it once fixed")
class Bnb4BitGPT2Test(Bnb4BitTest):
model_name = "openai-community/gpt2-xl"
EXPECTED_RELATIVE_DIFFERENCE = 3.3191854854152187

9
tests/quantization/bnb/test_mixed_int8.py
View File

@ -889,6 +889,7 @@ class MixedInt8TestTraining(BaseMixedInt8Test):
# Step 1: freeze all parameters
model = AutoModelForCausalLM.from_pretrained(self.model_name, load_in_8bit=True)
model.train()
if torch.cuda.is_available():
self.assertEqual(set(model.hf_device_map.values()), {torch.cuda.current_device()})
@ -914,14 +915,9 @@ class MixedInt8TestTraining(BaseMixedInt8Test):
batch = self.tokenizer("Test batch ", return_tensors="pt").to(torch_device)
# Step 4: Check if the gradient is not None
if torch_device in {"xpu", "cpu"}:
# XPU and CPU finetune do not support autocast for now.
with torch.autocast(torch_device):
out = model.forward(**batch)
out.logits.norm().backward()
else:
with torch.autocast(torch_device):
out = model.forward(**batch)
out.logits.norm().backward()
for module in model.modules():
if isinstance(module, LoRALayer):
@ -932,7 +928,6 @@ class MixedInt8TestTraining(BaseMixedInt8Test):
@apply_skip_if_not_implemented
@unittest.skipIf(torch_device == "xpu", reason="XPU has precision issue on gpt model, will test it once fixed")
class MixedInt8GPT2Test(MixedInt8Test):
model_name = "openai-community/gpt2-xl"
EXPECTED_RELATIVE_DIFFERENCE = 1.8720077507258357