riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-24 14:58:56 +06:00
Code Issues Actions 22 Packages Projects Releases Wiki Activity

Give each test a different repo name (#11453)

Browse Source
This commit is contained in:
Sylvain Gugger 2021-04-26 11:52:23 -04:00 committed by GitHub
parent b03b2a653d
commit 7959d83599
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 30 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
tests/test_configuration_common.py
View File

@ -99,12 +99,12 @@ class ConfigPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
cls._api.delete_repo(token=cls._token, name="test-model") cls._api.delete_repo(token=cls._token, name="test-config")
except HTTPError: except HTTPError:
pass pass
try: try:
cls._api.delete_repo(token=cls._token, name="test-model-org", organization="valid_org") cls._api.delete_repo(token=cls._token, name="test-config-org", organization="valid_org")
except HTTPError: except HTTPError:
pass pass
@ -113,9 +113,9 @@ class ConfigPushToHubTester(unittest.TestCase):
vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37 vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37
) )
with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.TemporaryDirectory() as tmp_dir:
config.save_pretrained(tmp_dir, push_to_hub=True, repo_name="test-model", use_auth_token=self._token) config.save_pretrained(tmp_dir, push_to_hub=True, repo_name="test-config", use_auth_token=self._token)
new_config = BertConfig.from_pretrained(f"{USER}/test-model") new_config = BertConfig.from_pretrained(f"{USER}/test-config")
for k, v in config.__dict__.items(): for k, v in config.__dict__.items():
if k != "transformers_version": if k != "transformers_version":
self.assertEqual(v, getattr(new_config, k)) self.assertEqual(v, getattr(new_config, k))
@ -129,12 +129,12 @@ class ConfigPushToHubTester(unittest.TestCase):
config.save_pretrained( config.save_pretrained(
tmp_dir, tmp_dir,
push_to_hub=True, push_to_hub=True,
repo_name="test-model-org", repo_name="test-config-org",
use_auth_token=self._token, use_auth_token=self._token,
organization="valid_org", organization="valid_org",
) )
new_config = BertConfig.from_pretrained("valid_org/test-model-org") new_config = BertConfig.from_pretrained("valid_org/test-config-org")
for k, v in config.__dict__.items(): for k, v in config.__dict__.items():
if k != "transformers_version": if k != "transformers_version":
self.assertEqual(v, getattr(new_config, k)) self.assertEqual(v, getattr(new_config, k))

12
tests/test_modeling_tf_common.py
View File

@ -1357,12 +1357,12 @@ class TFModelPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
cls._api.delete_repo(token=cls._token, name="test-model") cls._api.delete_repo(token=cls._token, name="test-model-tf")
except HTTPError: except HTTPError:
pass pass
try: try:
cls._api.delete_repo(token=cls._token, name="test-model-org", organization="valid_org") cls._api.delete_repo(token=cls._token, name="test-model-tf-org", organization="valid_org")
except HTTPError: except HTTPError:
pass pass
@ -1374,9 +1374,9 @@ class TFModelPushToHubTester(unittest.TestCase):
# Make sure model is properly initialized # Make sure model is properly initialized
_ = model(model.dummy_inputs) _ = model(model.dummy_inputs)
with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.TemporaryDirectory() as tmp_dir:
model.save_pretrained(tmp_dir, push_to_hub=True, repo_name="test-model", use_auth_token=self._token) model.save_pretrained(tmp_dir, push_to_hub=True, repo_name="test-model-tf", use_auth_token=self._token)
new_model = TFBertModel.from_pretrained(f"{USER}/test-model") new_model = TFBertModel.from_pretrained(f"{USER}/test-model-tf")
models_equal = True models_equal = True
for p1, p2 in zip(model.weights, new_model.weights): for p1, p2 in zip(model.weights, new_model.weights):
if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0:
@ -1392,12 +1392,12 @@ class TFModelPushToHubTester(unittest.TestCase):
model.save_pretrained( model.save_pretrained(
tmp_dir, tmp_dir,
push_to_hub=True, push_to_hub=True,
repo_name="test-model-org", repo_name="test-model-tf-org",
use_auth_token=self._token, use_auth_token=self._token,
organization="valid_org", organization="valid_org",
) )
new_model = TFBertModel.from_pretrained("valid_org/test-model-org") new_model = TFBertModel.from_pretrained("valid_org/test-model-tf-org")
models_equal = True models_equal = True
for p1, p2 in zip(model.weights, new_model.weights): for p1, p2 in zip(model.weights, new_model.weights):
if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0:

16
tests/test_tokenization_common.py
View File

@ -2874,7 +2874,7 @@ class TokenizerTesterMixin:
@is_staging_test @is_staging_test
class TokenzierPushToHubTester(unittest.TestCase): class TokenizerPushToHubTester(unittest.TestCase):
vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"] vocab_tokens = ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]
@classmethod @classmethod
@ -2885,12 +2885,12 @@ class TokenzierPushToHubTester(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
cls._api.delete_repo(token=cls._token, name="test-model") cls._api.delete_repo(token=cls._token, name="test-tokenizer")
except HTTPError: except HTTPError:
pass pass
try: try:
cls._api.delete_repo(token=cls._token, name="test-model-org", organization="valid_org") cls._api.delete_repo(token=cls._token, name="test-tokenizer-org", organization="valid_org")
except HTTPError: except HTTPError:
pass pass
@ -2900,9 +2900,11 @@ class TokenzierPushToHubTester(unittest.TestCase):
with open(vocab_file, "w", encoding="utf-8") as vocab_writer: with open(vocab_file, "w", encoding="utf-8") as vocab_writer:
vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens])) vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]))
tokenizer = BertTokenizer(vocab_file) tokenizer = BertTokenizer(vocab_file)
tokenizer.save_pretrained(tmp_dir, push_to_hub=True, repo_name="test-model", use_auth_token=self._token) tokenizer.save_pretrained(
tmp_dir, push_to_hub=True, repo_name="test-tokenizer", use_auth_token=self._token
)
new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-model") new_tokenizer = BertTokenizer.from_pretrained(f"{USER}/test-tokenizer")
self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab)
def test_push_to_hub_in_organization(self): def test_push_to_hub_in_organization(self):
@ -2914,10 +2916,10 @@ class TokenzierPushToHubTester(unittest.TestCase):
tokenizer.save_pretrained( tokenizer.save_pretrained(
tmp_dir, tmp_dir,
push_to_hub=True, push_to_hub=True,
repo_name="test-model-org", repo_name="test-tokenizer-org",
use_auth_token=self._token, use_auth_token=self._token,
organization="valid_org", organization="valid_org",
) )
new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-model-org") new_tokenizer = BertTokenizer.from_pretrained("valid_org/test-tokenizer-org")
self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab) self.assertDictEqual(new_tokenizer.vocab, tokenizer.vocab)

16
tests/test_trainer.py
View File

@ -1099,12 +1099,12 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
@classmethod @classmethod
def tearDownClass(cls): def tearDownClass(cls):
try: try:
cls._api.delete_repo(token=cls._token, name="test-model") cls._api.delete_repo(token=cls._token, name="test-trainer")
except HTTPError: except HTTPError:
pass pass
try: try:
cls._api.delete_repo(token=cls._token, name="test-model-org", organization="valid_org") cls._api.delete_repo(token=cls._token, name="test-trainer-org", organization="valid_org")
except HTTPError: except HTTPError:
pass pass
@ -1112,14 +1112,14 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.TemporaryDirectory() as tmp_dir:
trainer = get_regression_trainer(output_dir=tmp_dir) trainer = get_regression_trainer(output_dir=tmp_dir)
trainer.save_model() trainer.save_model()
url = trainer.push_to_hub(repo_name="test-model", use_auth_token=self._token) url = trainer.push_to_hub(repo_name="test-trainer", use_auth_token=self._token)
# Extract repo_name from the url # Extract repo_name from the url
re_search = re.search(ENDPOINT_STAGING + r"/([^/]+/[^/]+)/", url) re_search = re.search(ENDPOINT_STAGING + r"/([^/]+/[^/]+)/", url)
self.assertTrue(re_search is not None) self.assertTrue(re_search is not None)
repo_name = re_search.groups()[0] repo_name = re_search.groups()[0]
self.assertEqual(repo_name, f"{USER}/test-model") self.assertEqual(repo_name, f"{USER}/test-trainer")
model = RegressionPreTrainedModel.from_pretrained(repo_name) model = RegressionPreTrainedModel.from_pretrained(repo_name)
self.assertEqual(model.a.item(), trainer.model.a.item()) self.assertEqual(model.a.item(), trainer.model.a.item())
@ -1129,15 +1129,17 @@ class TrainerIntegrationWithHubTester(unittest.TestCase):
with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.TemporaryDirectory() as tmp_dir:
trainer = get_regression_trainer(output_dir=tmp_dir) trainer = get_regression_trainer(output_dir=tmp_dir)
trainer.save_model() trainer.save_model()
url = trainer.push_to_hub(repo_name="test-model-org", organization="valid_org", use_auth_token=self._token) url = trainer.push_to_hub(
repo_name="test-trainer-org", organization="valid_org", use_auth_token=self._token
)
# Extract repo_name from the url # Extract repo_name from the url
re_search = re.search(ENDPOINT_STAGING + r"/([^/]+/[^/]+)/", url) re_search = re.search(ENDPOINT_STAGING + r"/([^/]+/[^/]+)/", url)
self.assertTrue(re_search is not None) self.assertTrue(re_search is not None)
repo_name = re_search.groups()[0] repo_name = re_search.groups()[0]
self.assertEqual(repo_name, "valid_org/test-model-org") self.assertEqual(repo_name, "valid_org/test-trainer-org")
model = RegressionPreTrainedModel.from_pretrained("valid_org/test-model-org") model = RegressionPreTrainedModel.from_pretrained("valid_org/test-trainer-org")
self.assertEqual(model.a.item(), trainer.model.a.item()) self.assertEqual(model.a.item(), trainer.model.a.item())
self.assertEqual(model.b.item(), trainer.model.b.item()) self.assertEqual(model.b.item(), trainer.model.b.item())