diff --git a/examples/pytorch/token-classification/run_ner_no_trainer.py b/examples/pytorch/token-classification/run_ner_no_trainer.py index 84f506a42ca..cd8f7658d98 100755 --- a/examples/pytorch/token-classification/run_ner_no_trainer.py +++ b/examples/pytorch/token-classification/run_ner_no_trainer.py @@ -28,6 +28,7 @@ from pathlib import Path import datasets import evaluate +import numpy as np import torch from accelerate import Accelerator from accelerate.logging import get_logger @@ -777,6 +778,12 @@ def main(): if args.with_tracking: all_results.update({"train_loss": total_loss.item() / len(train_dataloader)}) with open(os.path.join(args.output_dir, "all_results.json"), "w") as f: + # Convert all float64 & int64 type numbers to float & int for json serialization + for key, value in all_results.items(): + if isinstance(value, np.float64): + all_results[key] = float(value) + elif isinstance(value, np.int64): + all_results[key] = int(value) json.dump(all_results, f) diff --git a/examples/research_projects/codeparrot/scripts/human_eval.py b/examples/research_projects/codeparrot/scripts/human_eval.py index 157079881d5..ef217a597e3 100644 --- a/examples/research_projects/codeparrot/scripts/human_eval.py +++ b/examples/research_projects/codeparrot/scripts/human_eval.py @@ -60,7 +60,7 @@ class EndOfFunctionCriteria(StoppingCriteria): decoded_generations = self.tokenizer.batch_decode(input_ids[:, self.start_length :]) done = [] for decoded_generation in decoded_generations: - done.append(any([stop_string in decoded_generation for stop_string in self.eof_strings])) + done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings)) return all(done) diff --git a/examples/research_projects/fsner/src/fsner/tokenizer_utils.py b/examples/research_projects/fsner/src/fsner/tokenizer_utils.py index bc5f6650ccd..b281ae6cfb8 100644 --- a/examples/research_projects/fsner/src/fsner/tokenizer_utils.py +++ b/examples/research_projects/fsner/src/fsner/tokenizer_utils.py @@ -17,7 +17,7 @@ class FSNERTokenizerUtils(object): `transformers.tokenization_utils_base.BatchEncoding` dict with additional keys and values for start_token_id, end_token_id and sizes of example lists for each entity type """ - if isinstance(x, list) and all([isinstance(_x, list) for _x in x]): + if isinstance(x, list) and all(isinstance(_x, list) for _x in x): d = None for l in x: t = self.tokenizer( @@ -37,7 +37,7 @@ class FSNERTokenizerUtils(object): d["start_token_id"] = torch.tensor(self.tokenizer.convert_tokens_to_ids("[E]")) d["end_token_id"] = torch.tensor(self.tokenizer.convert_tokens_to_ids("[/E]")) - elif isinstance(x, list) and all([isinstance(_x, str) for _x in x]): + elif isinstance(x, list) and all(isinstance(_x, str) for _x in x): d = self.tokenizer( x, padding="max_length", diff --git a/examples/research_projects/jax-projects/big_bird/prepare_natural_questions.py b/examples/research_projects/jax-projects/big_bird/prepare_natural_questions.py index 6a202ba7752..ebbb184ccb6 100644 --- a/examples/research_projects/jax-projects/big_bird/prepare_natural_questions.py +++ b/examples/research_projects/jax-projects/big_bird/prepare_natural_questions.py @@ -50,7 +50,7 @@ def _get_single_answer(example): answer["remove_it"] = False cols = ["start_token", "end_token", "start_byte", "end_byte", "text"] - if not all([isinstance(answer[k], list) for k in cols]): + if not all(isinstance(answer[k], list) for k in cols): raise ValueError("Issue in ID", example["id"]) return answer diff --git a/examples/research_projects/luke/run_luke_ner_no_trainer.py b/examples/research_projects/luke/run_luke_ner_no_trainer.py index 4c5227d2c7e..6b59643cf70 100644 --- a/examples/research_projects/luke/run_luke_ner_no_trainer.py +++ b/examples/research_projects/luke/run_luke_ner_no_trainer.py @@ -610,7 +610,7 @@ def main(): predicted_sequence = [label_list[0]] * len(true_tags) for _, span, label in sorted(predictions, key=lambda o: o[0], reverse=True): - if all([o == label_list[0] for o in predicted_sequence[span[0] : span[1]]]): + if all(o == label_list[0] for o in predicted_sequence[span[0] : span[1]]): predicted_sequence[span[0]] = label if span[1] - span[0] > 1: predicted_sequence[span[0] + 1 : span[1]] = [label] * (span[1] - span[0] - 1) diff --git a/examples/research_projects/lxmert/modeling_frcnn.py b/examples/research_projects/lxmert/modeling_frcnn.py index edbd224cbe0..943588a5ed8 100644 --- a/examples/research_projects/lxmert/modeling_frcnn.py +++ b/examples/research_projects/lxmert/modeling_frcnn.py @@ -554,8 +554,8 @@ class Matcher(object): assert thresholds[0] > 0 thresholds.insert(0, -float("inf")) thresholds.append(float("inf")) - assert all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]) - assert all([label_i in [-1, 0, 1] for label_i in labels]) + assert all(low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])) + assert all(label_i in [-1, 0, 1] for label_i in labels) assert len(labels) == len(thresholds) - 1 self.thresholds = thresholds self.labels = labels diff --git a/examples/research_projects/visual_bert/modeling_frcnn.py b/examples/research_projects/visual_bert/modeling_frcnn.py index edbd224cbe0..943588a5ed8 100644 --- a/examples/research_projects/visual_bert/modeling_frcnn.py +++ b/examples/research_projects/visual_bert/modeling_frcnn.py @@ -554,8 +554,8 @@ class Matcher(object): assert thresholds[0] > 0 thresholds.insert(0, -float("inf")) thresholds.append(float("inf")) - assert all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]) - assert all([label_i in [-1, 0, 1] for label_i in labels]) + assert all(low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])) + assert all(label_i in [-1, 0, 1] for label_i in labels) assert len(labels) == len(thresholds) - 1 self.thresholds = thresholds self.labels = labels diff --git a/src/transformers/generation/logits_process.py b/src/transformers/generation/logits_process.py index 87a3b17bcb1..952e86570a9 100644 --- a/src/transformers/generation/logits_process.py +++ b/src/transformers/generation/logits_process.py @@ -110,7 +110,7 @@ class MinLengthLogitsProcessor(LogitsProcessor): if isinstance(eos_token_id, int): eos_token_id = [eos_token_id] - if not all([isinstance(i, int) for i in eos_token_id]) or any([i < 0 for i in eos_token_id]): + if not all(isinstance(i, int) for i in eos_token_id) or any(i < 0 for i in eos_token_id): logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") self.min_length = min_length @@ -147,7 +147,7 @@ class MinNewTokensLengthLogitsProcessor(LogitsProcessor): if isinstance(eos_token_id, int): eos_token_id = [eos_token_id] - if not all([isinstance(i, int) for i in eos_token_id]) or any([i < 0 for i in eos_token_id]): + if not all(isinstance(i, int) for i in eos_token_id) or any(i < 0 for i in eos_token_id): logger.warning(f"`eos_token_id` has to be a list of positive integers, but is {eos_token_id}") self.prompt_length_to_skip = prompt_length_to_skip @@ -731,7 +731,7 @@ class NoBadWordsLogitsProcessor(SequenceBiasLogitsProcessor): if isinstance(eos_token_id, int): eos_token_id = [eos_token_id] bad_words_ids = list( - filter(lambda bad_token_seq: all([bad_token_seq != [i] for i in eos_token_id]), bad_words_ids) + filter(lambda bad_token_seq: all(bad_token_seq != [i] for i in eos_token_id), bad_words_ids) ) # Forbidding a sequence is equivalent to setting its bias to -inf diff --git a/src/transformers/generation/tf_logits_process.py b/src/transformers/generation/tf_logits_process.py index 2c589ca98c7..7e442a1659c 100644 --- a/src/transformers/generation/tf_logits_process.py +++ b/src/transformers/generation/tf_logits_process.py @@ -318,7 +318,7 @@ class TFNoBadWordsLogitsProcessor(TFLogitsProcessor): self.bad_word_seqs_ids = tf.ragged.constant(bad_words_ids).to_tensor(default_value=-1) # 2. a tensor with the unpadded length of each forbidden sequence, for quick length comparisons bad_word_seqs_len = [len(bad_words) for bad_words in bad_words_ids] - if any([word_len == 0 for word_len in bad_word_seqs_len]): + if any(word_len == 0 for word_len in bad_word_seqs_len): raise ValueError(f"Banned words token sequences {bad_words_ids} cannot have an empty list") self.bad_word_seqs_len = tf.convert_to_tensor(bad_word_seqs_len, dtype=tf.int32) # 3. a tensor containing the last token for each sequence, for easy access to the tokens that may be banned diff --git a/src/transformers/generation/tf_utils.py b/src/transformers/generation/tf_utils.py index ff8be3dc0ae..40c418a714a 100644 --- a/src/transformers/generation/tf_utils.py +++ b/src/transformers/generation/tf_utils.py @@ -1638,7 +1638,7 @@ class TFGenerationMixin: # TODO (Joao): fix cache format or find programatic way to detect cache index # GPT2 and other models has a slightly different cache structure, with a different batch axis model_name = str(self.decoder) if "EncoderDecoder" in str(self) else str(self) - cache_batch_axis = 1 if any([model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")]) else 0 + cache_batch_axis = 1 if any(model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")) else 0 # some models, like XLNet, need more than the last token in the presence of past_key_values needs_full_input = "use_mems" in set(inspect.signature(self.prepare_inputs_for_generation).parameters.keys()) @@ -1922,7 +1922,7 @@ class TFGenerationMixin: # TODO (Joao): fix cache format or find programatic way to detect cache index # GPT2 and other models has a slightly different cache structure, with a different batch axis model_name = str(self.decoder) if "EncoderDecoder" in str(self) else str(self) - cache_batch_axis = 1 if any([model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")]) else 0 + cache_batch_axis = 1 if any(model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")) else 0 # some models, like XLNet, need more than the last token in the presence of past_key_values needs_full_input = "use_mems" in set(inspect.signature(self.prepare_inputs_for_generation).parameters.keys()) @@ -2265,7 +2265,7 @@ class TFGenerationMixin: # TODO (Joao): fix cache format or find programatic way to detect cache index # GPT2 and other models has a slightly different cache structure, with a different batch axis model_name = str(self.decoder) if "EncoderDecoder" in str(self) else str(self) - cache_batch_axis = 1 if any([model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")]) else 0 + cache_batch_axis = 1 if any(model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")) else 0 # some models, like XLNet, need more than the last token in the presence of past_key_values needs_full_input = "use_mems" in set(inspect.signature(self.prepare_inputs_for_generation).parameters.keys()) @@ -2779,7 +2779,7 @@ class TFGenerationMixin: # TODO (Joao): fix cache format or find programatic way to detect cache index # GPT2 and other models has a slightly different cache structure, with a different batch axis model_name = str(self.decoder) if "EncoderDecoder" in str(self) else str(self) - cache_batch_axis = 1 if any([model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")]) else 0 + cache_batch_axis = 1 if any(model_prefix in model_name for model_prefix in ("TFGPT2", "TFCTRL")) else 0 # 2. init `attentions`, `hidden_states`, and `scores` tuples scores = [] if (return_dict_in_generate and output_scores) else None diff --git a/src/transformers/keras_callbacks.py b/src/transformers/keras_callbacks.py index 661a8c09d58..22e1ff682cc 100644 --- a/src/transformers/keras_callbacks.py +++ b/src/transformers/keras_callbacks.py @@ -144,7 +144,7 @@ class KerasMetricCallback(Callback): @staticmethod def _concatenate_batches(batches, padding_index=-100): # If all batches are unidimensional or same length, do a simple concatenation - if batches[0].ndim == 1 or all([batch.shape[1] == batches[0].shape[1] for batch in batches]): + if batches[0].ndim == 1 or all(batch.shape[1] == batches[0].shape[1] for batch in batches): return np.concatenate(batches, axis=0) # Welp, they're not the same length. Let's do some padding diff --git a/src/transformers/models/bert/convert_bert_pytorch_checkpoint_to_original_tf.py b/src/transformers/models/bert/convert_bert_pytorch_checkpoint_to_original_tf.py index 68ed9bafc87..5e3ef4df9fe 100644 --- a/src/transformers/models/bert/convert_bert_pytorch_checkpoint_to_original_tf.py +++ b/src/transformers/models/bert/convert_bert_pytorch_checkpoint_to_original_tf.py @@ -78,7 +78,7 @@ def convert_pytorch_checkpoint_to_tf(model: BertModel, ckpt_dir: str, model_name for var_name in state_dict: tf_name = to_tf_var_name(var_name) torch_tensor = state_dict[var_name].numpy() - if any([x in var_name for x in tensors_to_transpose]): + if any(x in var_name for x in tensors_to_transpose): torch_tensor = torch_tensor.T tf_var = create_tf_var(tensor=torch_tensor, name=tf_name, session=session) tf.keras.backend.set_value(tf_var, torch_tensor) diff --git a/src/transformers/models/bigbird_pegasus/convert_bigbird_pegasus_tf_to_pytorch.py b/src/transformers/models/bigbird_pegasus/convert_bigbird_pegasus_tf_to_pytorch.py index 5a81207548f..e17369e4804 100644 --- a/src/transformers/models/bigbird_pegasus/convert_bigbird_pegasus_tf_to_pytorch.py +++ b/src/transformers/models/bigbird_pegasus/convert_bigbird_pegasus_tf_to_pytorch.py @@ -104,7 +104,7 @@ def convert_bigbird_pegasus(tf_weights: dict, config_update: dict) -> BigBirdPeg new_k = rename_state_dict_key(k, patterns) if new_k not in state_dict: raise ValueError(f"could not find new key {new_k} in state dict. (converted from {k})") - if any([True if i in k else False for i in ["dense", "query", "key", "value"]]): + if any(True if i in k else False for i in ["dense", "query", "key", "value"]): v = v.T mapping[new_k] = torch.from_numpy(v) assert v.shape == state_dict[new_k].shape, f"{new_k}, {k}, {v.shape}, {state_dict[new_k].shape}" @@ -117,7 +117,7 @@ def convert_bigbird_pegasus(tf_weights: dict, config_update: dict) -> BigBirdPeg new_k = rename_state_dict_key(k, patterns) if new_k not in state_dict and k != "pegasus/embeddings/position_embeddings": raise ValueError(f"could not find new key {new_k} in state dict. (converted from {k})") - if any([True if i in k else False for i in ["dense", "query", "key", "value"]]): + if any(True if i in k else False for i in ["dense", "query", "key", "value"]): v = v.T mapping[new_k] = torch.from_numpy(v) if k != "pegasus/embeddings/position_embeddings": @@ -147,7 +147,7 @@ def get_tf_weights_as_numpy(path) -> Dict: tf_weights = {} ignore_name = ["global_step"] for name, shape in tqdm(init_vars, desc="converting tf checkpoint to dict"): - skip_key = any([pat in name for pat in ignore_name]) + skip_key = any(pat in name for pat in ignore_name) if skip_key: continue array = tf.train.load_variable(path, name) diff --git a/src/transformers/models/deta/modeling_deta.py b/src/transformers/models/deta/modeling_deta.py index af218829d6f..bee84a5bf72 100644 --- a/src/transformers/models/deta/modeling_deta.py +++ b/src/transformers/models/deta/modeling_deta.py @@ -2485,9 +2485,9 @@ class DetaMatcher(object): thresholds.insert(0, -float("inf")) thresholds.append(float("inf")) # Currently torchscript does not support all + generator - if not all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]): + if not all(low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])): raise ValueError("Thresholds should be sorted.") - if not all([l in [-1, 0, 1] for l in labels]): + if not all(l in [-1, 0, 1] for l in labels): raise ValueError("All labels should be either -1, 0 or 1") if len(labels) != len(thresholds) - 1: raise ValueError("Number of labels should be equal to number of thresholds - 1") diff --git a/src/transformers/models/dpr/tokenization_dpr.py b/src/transformers/models/dpr/tokenization_dpr.py index a2024dda5d6..b2ae84addc7 100644 --- a/src/transformers/models/dpr/tokenization_dpr.py +++ b/src/transformers/models/dpr/tokenization_dpr.py @@ -379,11 +379,9 @@ class CustomDPRReaderTokenizerMixin: if length > max_answer_length: raise ValueError(f"Span is too long: {length} > {max_answer_length}") if any( - [ - start_index <= prev_start_index <= prev_end_index <= end_index - or prev_start_index <= start_index <= end_index <= prev_end_index - for (prev_start_index, prev_end_index) in chosen_span_intervals - ] + start_index <= prev_start_index <= prev_end_index <= end_index + or prev_start_index <= start_index <= end_index <= prev_end_index + for (prev_start_index, prev_end_index) in chosen_span_intervals ): continue chosen_span_intervals.append((start_index, end_index)) diff --git a/src/transformers/models/dpr/tokenization_dpr_fast.py b/src/transformers/models/dpr/tokenization_dpr_fast.py index de32332bf27..784ed1344cf 100644 --- a/src/transformers/models/dpr/tokenization_dpr_fast.py +++ b/src/transformers/models/dpr/tokenization_dpr_fast.py @@ -377,11 +377,9 @@ class CustomDPRReaderTokenizerMixin: length = end_index - start_index + 1 assert length <= max_answer_length, f"Span is too long: {length} > {max_answer_length}" if any( - [ - start_index <= prev_start_index <= prev_end_index <= end_index - or prev_start_index <= start_index <= end_index <= prev_end_index - for (prev_start_index, prev_end_index) in chosen_span_intervals - ] + start_index <= prev_start_index <= prev_end_index <= end_index + or prev_start_index <= start_index <= end_index <= prev_end_index + for (prev_start_index, prev_end_index) in chosen_span_intervals ): continue chosen_span_intervals.append((start_index, end_index)) diff --git a/src/transformers/models/pegasus/convert_pegasus_tf_to_pytorch.py b/src/transformers/models/pegasus/convert_pegasus_tf_to_pytorch.py index 739e075233f..cf183b590c1 100644 --- a/src/transformers/models/pegasus/convert_pegasus_tf_to_pytorch.py +++ b/src/transformers/models/pegasus/convert_pegasus_tf_to_pytorch.py @@ -90,7 +90,7 @@ def get_tf_weights_as_numpy(path="./ckpt/aeslc/model.ckpt-32000") -> Dict: tf_weights = {} ignore_name = ["Adafactor", "global_step"] for name, shape in tqdm(init_vars, desc="converting tf checkpoint to dict"): - skip_key = any([pat in name for pat in ignore_name]) + skip_key = any(pat in name for pat in ignore_name) if skip_key: continue array = tf.train.load_variable(path, name) diff --git a/src/transformers/models/sam/processing_sam.py b/src/transformers/models/sam/processing_sam.py index fd73260f940..0ec47a995af 100644 --- a/src/transformers/models/sam/processing_sam.py +++ b/src/transformers/models/sam/processing_sam.py @@ -115,7 +115,7 @@ class SamProcessor(ProcessorMixin): for point, original_size in zip(input_points, original_sizes) ] # check that all arrays have the same shape - if not all([point.shape == input_points[0].shape for point in input_points]): + if not all(point.shape == input_points[0].shape for point in input_points): if input_labels is not None: input_points, input_labels = self._pad_points_and_labels(input_points, input_labels) diff --git a/tests/generation/test_framework_agnostic.py b/tests/generation/test_framework_agnostic.py index 61845aa9bc1..306cb15168e 100644 --- a/tests/generation/test_framework_agnostic.py +++ b/tests/generation/test_framework_agnostic.py @@ -647,7 +647,7 @@ class GenerationIntegrationTestsMixin: generated_tokens = model.generate(**tokens, eos_token_id=eos_token_id, **generation_kwargs) unpadded_correct_condition = expectation == len(generated_tokens[0]) padded_correct_condition = expectation < len(generated_tokens[0]) and all( - [token == model.config.pad_token_id for token in generated_tokens[0][expectation:]] + token == model.config.pad_token_id for token in generated_tokens[0][expectation:] ) self.assertTrue(unpadded_correct_condition or padded_correct_condition) @@ -655,7 +655,7 @@ class GenerationIntegrationTestsMixin: generated_tokens = model.generate(**tokens, eos_token_id=eos_token_id, **generation_kwargs) unpadded_correct_condition = expectation == len(generated_tokens[0]) padded_correct_condition = expectation < len(generated_tokens[0]) and all( - [token == model.config.pad_token_id for token in generated_tokens[0][expectation:]] + token == model.config.pad_token_id for token in generated_tokens[0][expectation:] ) self.assertTrue(unpadded_correct_condition or padded_correct_condition) diff --git a/tests/models/codegen/test_modeling_codegen.py b/tests/models/codegen/test_modeling_codegen.py index fdf2d891929..9072c2b5bc2 100644 --- a/tests/models/codegen/test_modeling_codegen.py +++ b/tests/models/codegen/test_modeling_codegen.py @@ -521,7 +521,7 @@ class CodeGenModelLanguageGenerationTest(unittest.TestCase): self.assertEqual(output_str, EXPECTED_OUTPUT_STR) self.assertTrue( - all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))]) + all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))) ) # token_type_ids should change output @is_flaky(max_attempts=3, description="measure of timing is somehow flaky.") diff --git a/tests/models/data2vec/test_modeling_data2vec_audio.py b/tests/models/data2vec/test_modeling_data2vec_audio.py index 74450f50c8b..67fe0cbe70c 100644 --- a/tests/models/data2vec/test_modeling_data2vec_audio.py +++ b/tests/models/data2vec/test_modeling_data2vec_audio.py @@ -516,7 +516,7 @@ class Data2VecAudioModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Tes "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/encodec/test_modeling_encodec.py b/tests/models/encodec/test_modeling_encodec.py index a1693b75824..427f92f6c25 100644 --- a/tests/models/encodec/test_modeling_encodec.py +++ b/tests/models/encodec/test_modeling_encodec.py @@ -373,12 +373,12 @@ class EncodecModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase) uniform_init_parms = ["conv"] ignore_init = ["lstm"] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", ) - elif not any([x in name for x in ignore_init]): + elif not any(x in name for x in ignore_init): self.assertIn( ((param.data.mean() * 1e9).round() / 1e9).item(), [0.0, 1.0], diff --git a/tests/models/gpt2/test_modeling_gpt2.py b/tests/models/gpt2/test_modeling_gpt2.py index 65542b49549..f820b54942e 100644 --- a/tests/models/gpt2/test_modeling_gpt2.py +++ b/tests/models/gpt2/test_modeling_gpt2.py @@ -768,7 +768,7 @@ class GPT2ModelLanguageGenerationTest(unittest.TestCase): ) self.assertEqual(output_str, EXPECTED_OUTPUT_STR) self.assertTrue( - all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))]) + all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))) ) # token_type_ids should change output @slow diff --git a/tests/models/gptj/test_modeling_gptj.py b/tests/models/gptj/test_modeling_gptj.py index 7fd6a40e173..3636d357d5d 100644 --- a/tests/models/gptj/test_modeling_gptj.py +++ b/tests/models/gptj/test_modeling_gptj.py @@ -571,7 +571,7 @@ class GPTJModelLanguageGenerationTest(unittest.TestCase): self.assertEqual(output_str, EXPECTED_OUTPUT_STR) self.assertTrue( - all([output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))]) + all(output_seq_strs[idx] != output_seq_tt_strs[idx] for idx in range(len(output_seq_tt_strs))) ) # token_type_ids should change output @slow diff --git a/tests/models/hubert/test_modeling_hubert.py b/tests/models/hubert/test_modeling_hubert.py index 9b18b5580a8..bad1a561da0 100644 --- a/tests/models/hubert/test_modeling_hubert.py +++ b/tests/models/hubert/test_modeling_hubert.py @@ -423,7 +423,7 @@ class HubertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): "quantizer.weight_proj.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -684,7 +684,7 @@ class HubertRobustModelTest(ModelTesterMixin, unittest.TestCase): "quantizer.weight_proj.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/mctct/test_modeling_mctct.py b/tests/models/mctct/test_modeling_mctct.py index 7fa9d99a3fa..21fadd633c3 100644 --- a/tests/models/mctct/test_modeling_mctct.py +++ b/tests/models/mctct/test_modeling_mctct.py @@ -386,7 +386,7 @@ class MCTCTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -533,7 +533,7 @@ class MCTCTRobustModelTest(ModelTesterMixin, unittest.TestCase): "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/rwkv/test_modeling_rwkv.py b/tests/models/rwkv/test_modeling_rwkv.py index 67430f6f3dc..2b9cc47133a 100644 --- a/tests/models/rwkv/test_modeling_rwkv.py +++ b/tests/models/rwkv/test_modeling_rwkv.py @@ -334,7 +334,7 @@ class RwkvModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin if param.requires_grad: # check if it's a ones like self.assertTrue(torch.allclose(param.data, torch.ones_like(param.data), atol=1e-5, rtol=1e-5)) - elif any([x in name for x in ["time_mix_key", "time_mix_receptance"]]): + elif any(x in name for x in ["time_mix_key", "time_mix_receptance"]): if param.requires_grad: self.assertInterval( param.data, diff --git a/tests/models/sew/test_modeling_sew.py b/tests/models/sew/test_modeling_sew.py index aa511e788b8..651600c4372 100644 --- a/tests/models/sew/test_modeling_sew.py +++ b/tests/models/sew/test_modeling_sew.py @@ -417,7 +417,7 @@ class SEWModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): "quantizer.weight_proj.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/sew_d/test_modeling_sew_d.py b/tests/models/sew_d/test_modeling_sew_d.py index c6ee5337190..9aa4b8edace 100644 --- a/tests/models/sew_d/test_modeling_sew_d.py +++ b/tests/models/sew_d/test_modeling_sew_d.py @@ -431,7 +431,7 @@ class SEWDModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): "quantizer.weight_proj.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/speecht5/test_modeling_speecht5.py b/tests/models/speecht5/test_modeling_speecht5.py index 8fbbee84f22..7532f132a79 100644 --- a/tests/models/speecht5/test_modeling_speecht5.py +++ b/tests/models/speecht5/test_modeling_speecht5.py @@ -583,7 +583,7 @@ class SpeechT5ForSpeechToTextTest(ModelTesterMixin, unittest.TestCase): "feature_projection.projection.bias", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -927,7 +927,7 @@ class SpeechT5ForTextToSpeechTest(ModelTesterMixin, unittest.TestCase): "conv.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -1337,7 +1337,7 @@ class SpeechT5ForSpeechToSpeechTest(ModelTesterMixin, unittest.TestCase): "feature_projection.projection.bias", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/unispeech/test_modeling_unispeech.py b/tests/models/unispeech/test_modeling_unispeech.py index 4d7967c484f..6d0bd1bf1f2 100644 --- a/tests/models/unispeech/test_modeling_unispeech.py +++ b/tests/models/unispeech/test_modeling_unispeech.py @@ -432,7 +432,7 @@ class UniSpeechRobustModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.T "feature_projection.projection.bias", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/unispeech_sat/test_modeling_unispeech_sat.py b/tests/models/unispeech_sat/test_modeling_unispeech_sat.py index 19d3dd849f5..a418a56dadb 100644 --- a/tests/models/unispeech_sat/test_modeling_unispeech_sat.py +++ b/tests/models/unispeech_sat/test_modeling_unispeech_sat.py @@ -484,7 +484,7 @@ class UniSpeechSatModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.Test "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -695,7 +695,7 @@ class UniSpeechSatRobustModelTest(ModelTesterMixin, unittest.TestCase): "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/wav2vec2/test_modeling_flax_wav2vec2.py b/tests/models/wav2vec2/test_modeling_flax_wav2vec2.py index 508d96ae10e..b9b52dc121e 100644 --- a/tests/models/wav2vec2/test_modeling_flax_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_flax_wav2vec2.py @@ -464,7 +464,7 @@ class FlaxWav2Vec2UtilsTest(unittest.TestCase): negative_indices = _sample_negative_indices(features.shape, num_negatives, attention_mask=attention_mask) # make sure that no padding tokens are sampled - self.assertTrue(all([idx not in negative_indices for idx in forbidden_indices])) + self.assertTrue(all(idx not in negative_indices for idx in forbidden_indices)) features = features.reshape(-1, hidden_size) # BTC => (BxT)C # take negative vectors from sampled indices diff --git a/tests/models/wav2vec2/test_modeling_wav2vec2.py b/tests/models/wav2vec2/test_modeling_wav2vec2.py index 65bfcb4451a..4db9b156db4 100644 --- a/tests/models/wav2vec2/test_modeling_wav2vec2.py +++ b/tests/models/wav2vec2/test_modeling_wav2vec2.py @@ -637,7 +637,7 @@ class Wav2Vec2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", @@ -971,7 +971,7 @@ class Wav2Vec2RobustModelTest(ModelTesterMixin, unittest.TestCase): "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/wav2vec2_conformer/test_modeling_wav2vec2_conformer.py b/tests/models/wav2vec2_conformer/test_modeling_wav2vec2_conformer.py index 200c863adbb..8c26268c6da 100644 --- a/tests/models/wav2vec2_conformer/test_modeling_wav2vec2_conformer.py +++ b/tests/models/wav2vec2_conformer/test_modeling_wav2vec2_conformer.py @@ -569,7 +569,7 @@ class Wav2Vec2ConformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/wavlm/test_modeling_wavlm.py b/tests/models/wavlm/test_modeling_wavlm.py index 67f380c239e..b04a96dd1ca 100644 --- a/tests/models/wavlm/test_modeling_wavlm.py +++ b/tests/models/wavlm/test_modeling_wavlm.py @@ -438,7 +438,7 @@ class WavLMModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase): "objective.weight", ] if param.requires_grad: - if any([x in name for x in uniform_init_parms]): + if any(x in name for x in uniform_init_parms): self.assertTrue( -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0, msg=f"Parameter {name} of model {model_class} seems not properly initialized", diff --git a/tests/models/whisper/test_modeling_whisper.py b/tests/models/whisper/test_modeling_whisper.py index b16587fd66f..20bbfbac325 100644 --- a/tests/models/whisper/test_modeling_whisper.py +++ b/tests/models/whisper/test_modeling_whisper.py @@ -1535,7 +1535,7 @@ class WhisperModelIntegrationTests(unittest.TestCase): text = processor.decode(output[0]) self.assertTrue(prompt in text) - self.assertTrue(all([token in text for token in expected_tokens])) + self.assertTrue(all(token in text for token in expected_tokens)) @slow def test_generate_with_prompt_ids_and_no_non_prompt_forced_decoder_ids(self): diff --git a/tests/onnx/test_onnx.py b/tests/onnx/test_onnx.py index db1fc6ac454..851934ef302 100644 --- a/tests/onnx/test_onnx.py +++ b/tests/onnx/test_onnx.py @@ -145,7 +145,7 @@ class OnnxExportTestCase(unittest.TestCase): # Assert all variables are present self.assertEqual(len(shapes), len(variable_names)) - self.assertTrue(all([var_name in shapes for var_name in variable_names])) + self.assertTrue(all(var_name in shapes for var_name in variable_names)) self.assertSequenceEqual(variable_names[:3], input_vars) self.assertSequenceEqual(variable_names[3:], output_vars) diff --git a/tests/test_modeling_tf_common.py b/tests/test_modeling_tf_common.py index 37429b73fd9..fe225e2723e 100644 --- a/tests/test_modeling_tf_common.py +++ b/tests/test_modeling_tf_common.py @@ -1566,7 +1566,7 @@ class TFModelTesterMixin: return_labels=True if "labels" in inspect.signature(model_class.call).parameters.keys() else False, ) if not any( - [tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor)] + tensor.dtype.is_integer for tensor in prepared_for_class.values() if isinstance(tensor, tf.Tensor) ): return # No integer inputs means no need for this test diff --git a/tests/test_tokenization_common.py b/tests/test_tokenization_common.py index 0656e382a8e..3b17c6ea4f6 100644 --- a/tests/test_tokenization_common.py +++ b/tests/test_tokenization_common.py @@ -79,7 +79,7 @@ SMALL_TRAINING_CORPUS = [ def filter_non_english(_, pretrained_name: str): """Filter all the model for non-english language""" - return not any([lang in pretrained_name for lang in NON_ENGLISH_TAGS]) + return not any(lang in pretrained_name for lang in NON_ENGLISH_TAGS) def filter_roberta_detectors(_, pretrained_name: str): diff --git a/tests/trainer/test_trainer_seq2seq.py b/tests/trainer/test_trainer_seq2seq.py index 6212dea1f7b..918c2215583 100644 --- a/tests/trainer/test_trainer_seq2seq.py +++ b/tests/trainer/test_trainer_seq2seq.py @@ -56,8 +56,8 @@ class Seq2seqTrainerTester(TestCasePlus): ] batch["decoder_attention_mask"] = outputs.attention_mask - assert all([len(x) == 512 for x in inputs.input_ids]) - assert all([len(x) == 128 for x in outputs.input_ids]) + assert all(len(x) == 512 for x in inputs.input_ids) + assert all(len(x) == 128 for x in outputs.input_ids) return batch diff --git a/utils/check_copies.py b/utils/check_copies.py index 0dbc7b889ed..959c7b2d329 100644 --- a/utils/check_copies.py +++ b/utils/check_copies.py @@ -362,7 +362,7 @@ def convert_to_localized_md(model_list, localized_model_list, format_str): model_keys = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in model_list.strip().split("\n")] # We exclude keys in localized README not in the main one. - readmes_match = not any([k not in model_keys for k in localized_model_index]) + readmes_match = not any(k not in model_keys for k in localized_model_index) localized_model_index = {k: v for k, v in localized_model_index.items() if k in model_keys} for model in model_list.strip().split("\n"): diff --git a/utils/create_dummy_models.py b/utils/create_dummy_models.py index 997aab3f602..0dfe5cea663 100644 --- a/utils/create_dummy_models.py +++ b/utils/create_dummy_models.py @@ -735,7 +735,7 @@ def build_model(model_arch, tiny_config, output_dir): tiny_config = copy.deepcopy(tiny_config) - if any([model_arch.__name__.endswith(x) for x in ["ForCausalLM", "LMHeadModel"]]): + if any(model_arch.__name__.endswith(x) for x in ["ForCausalLM", "LMHeadModel"]): tiny_config.is_encoder_decoder = False tiny_config.is_decoder = True diff --git a/utils/tests_fetcher.py b/utils/tests_fetcher.py index b6a1ec87ed7..88552a11ca1 100644 --- a/utils/tests_fetcher.py +++ b/utils/tests_fetcher.py @@ -428,7 +428,7 @@ def get_module_dependencies(module_fname, cache=None): # So we get the imports from that init then try to find where our objects come from. new_imported_modules = extract_imports(module, cache=cache) for new_module, new_imports in new_imported_modules: - if any([i in new_imports for i in imports]): + if any(i in new_imports for i in imports): if new_module not in dependencies: new_modules.append((new_module, [i for i in new_imports if i in imports])) imports = [i for i in imports if i not in new_imports]