update ruff version (#30932)

* update ruff version

* fix research projects

* Empty

* Fix errors

---------

Co-authored-by: Lysandre <lysandre@huggingface.co>
This commit is contained in:
Arthur 2024-05-22 06:40:15 +02:00 committed by GitHub
parent 60bb571e99
commit 673440d073
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1172 changed files with 1555 additions and 1861 deletions

View File

@ -5,7 +5,7 @@ export PYTHONPATH = src
check_dirs := examples tests src utils
exclude_folders := examples/research_projects
exclude_folders := ""
modified_only_fixup:
$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))

View File

@ -20,6 +20,7 @@ text file or a dataset.
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
https://huggingface.co/models?filter=fill-mask
"""
import json
import logging
import math

View File

@ -19,6 +19,7 @@ Pretraining the library models for T5-like span-masked language modeling on a te
Here is the full list of checkpoints on the hub that can be pretrained by this script:
https://huggingface.co/models?filter=t5
"""
import json
import logging
import math

View File

@ -15,6 +15,7 @@
"""
Post-processing utilities for question answering.
"""
import collections
import json
import logging

View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Finetuning a 🤗 Flax Transformers model for sequence classification on GLUE."""
import json
import logging
import math

View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fine-tuning a 🤗 Flax Transformers model on token classification tasks (NER, POS, CHUNKS)"""
import json
import logging
import math

View File

@ -93,14 +93,14 @@ class Plot:
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
if can_convert_to_int(row["result"]):
# value is not None
self.result_dict[model_name]["result"][
(int(row["batch_size"]), int(row["sequence_length"]))
] = int(row["result"])
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
int(row["result"])
)
elif can_convert_to_float(row["result"]):
# value is not None
self.result_dict[model_name]["result"][
(int(row["batch_size"]), int(row["sequence_length"]))
] = float(row["result"])
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
float(row["result"])
)
def plot(self):
fig, ax = plt.subplots()

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Finetuning the library models for multiple choice (Bert, Roberta, XLNet)."""
import logging
import os
from dataclasses import dataclass, field

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension"""
import csv
import glob
import json

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
import argparse
import glob
import logging

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Fine-tuning the library models for question-answering."""
import logging
import os
import sys

View File

@ -20,7 +20,6 @@ GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss.
using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.
"""
import logging
import math
import os

View File

@ -28,6 +28,7 @@
--output_dir ../log \
--train_batch_size 16 \
"""
import argparse
import csv
import logging

View File

@ -18,7 +18,6 @@
Finetuning the library models for multiple choice on SWAG (Bert).
"""
import argparse
import csv
import glob

View File

@ -21,7 +21,6 @@
This script with default values evaluates a pretrained Transformer-XL on WikiText 103
"""
import argparse
import logging
import math

View File

@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
"""
import importlib
import sys
from argparse import REMAINDER, ArgumentParser

View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Fine-tuning the library models for named entity recognition on CoNLL-2003."""
import logging
import os
import sys

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Named entity recognition fine-tuning: utilities to work with CoNLL-2003 task."""
import logging
import os
from dataclasses import dataclass

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Finetuning any 🤗 Transformers model for image classification leveraging 🤗 Accelerate."""
import argparse
import json
import logging

View File

@ -15,6 +15,7 @@
"""
A subclass of `Trainer` specific to Question-Answering tasks
"""
import math
import time

View File

@ -15,6 +15,7 @@
"""
A subclass of `Trainer` specific to Question-Answering tasks
"""
import math
import time
from typing import Dict, List, Optional

View File

@ -15,6 +15,7 @@
"""
Post-processing utilities for question answering.
"""
import collections
import json
import logging

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Finetuning a 🤗 Transformers model for sequence classification on GLUE."""
import argparse
import json
import logging

View File

@ -14,9 +14,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)
"""
"""Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)"""
import argparse
import inspect

View File

@ -19,7 +19,6 @@ Running this example:
python run_generation_contrastive_search.py --model_name_or_path=openai-community/gpt2-large --penalty_alpha=0.6 --k=4 --length=256
"""
import argparse
import logging

View File

@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
"""
import importlib
import sys
from argparse import REMAINDER, ArgumentParser

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""PyTorch BERT model with Patience-based Early Exit."""
import logging
import torch

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Training and inference using the library models for sequence classification on GLUE (Bert, Albert) with PABEE."""
import argparse
import glob
import json

View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""BertAbs configuration"""
import logging
from transformers import PretrainedConfig

View File

@ -19,6 +19,7 @@
Some parts of this script are adapted from the code of Michel et al. (http://arxiv.org/abs/1905.10650)
which is available at https://github.com/pmichel31415/are-16-heads-really-better-than-1
"""
import argparse
import logging
import os

View File

@ -15,6 +15,7 @@
"""The distiller to distil the student.
Adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
"""
import math
import os
import time

View File

@ -12,8 +12,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)
"""
"""Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)"""
import bisect
import copy
from collections import defaultdict

View File

@ -15,6 +15,7 @@
"""Dataset to distilled models
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
"""
import numpy as np
import torch
from torch.utils.data import Dataset

View File

@ -15,6 +15,7 @@
"""
Preprocessing script before distillation.
"""
import argparse
import logging
import pickle

View File

@ -16,6 +16,7 @@
Preprocessing script before training the distilled model.
Specific to RoBERTa -> DistilRoBERTa and GPT2 -> DistilGPT2.
"""
import argparse
import torch

View File

@ -16,6 +16,7 @@
Preprocessing script before training DistilBERT.
Specific to BERT -> DistilBERT.
"""
import argparse
import torch

View File

@ -15,6 +15,7 @@
"""
Preprocessing script before training the distilled model.
"""
import argparse
import logging
import pickle

View File

@ -16,6 +16,7 @@
Training the distilled model.
Supported architectures include: BERT -> DistilBERT, RoBERTa -> DistilRoBERTa, GPT2 -> DistilGPT2.
"""
import argparse
import json
import os

View File

@ -15,6 +15,7 @@
"""Utils to train DistilBERT
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
"""
import json
import logging
import os

View File

@ -20,6 +20,7 @@ text file or a dataset.
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
https://huggingface.co/models?filter=fill-mask
"""
import logging
import os
import sys

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import itertools
import math
import os

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import sys
from typing import Tuple

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import colorsys
import io

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Finetuning the library models for multimodal multiclass prediction on MM-IMDB dataset."""
import argparse
import glob
import json

View File

@ -15,6 +15,7 @@
Count remaining (non-zero) weights in the encoder (i.e. the transformer layers).
Sparsity and remaining weights levels are equivalent: sparsity % = 100 - remaining weights %.
"""
import argparse
import os

View File

@ -16,7 +16,6 @@
"""Masked BERT model configuration. It replicates the class `~transformers.BertConfig`
and adapts it to the specificities of MaskedBert (`pruning_method`, `mask_init` and `mask_scale`."""
import logging
from transformers.configuration_utils import PretrainedConfig

View File

@ -18,7 +18,6 @@
compute the adaptive mask.
Built on top of `transformers.models.bert.modeling_bert`"""
import logging
import math

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Fine-pruning Masked BERT for question-answering on SQuAD."""
import argparse
import glob
import logging

View File

@ -13,9 +13,8 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
""" """
"""
import argparse
import logging
import os

View File

@ -19,6 +19,7 @@ text file or a dataset.
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
https://huggingface.co/models?filter=fill-mask
"""
import logging
import os
import sys

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
import argparse
import logging
import os

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for training models with pytorch-quantization"""
import logging
import re

View File

@ -15,6 +15,7 @@
"""
Post-processing utilities for question answering.
"""
import collections
import json
import logging

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import itertools
import math
import os

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import sys
from typing import Tuple

View File

@ -15,6 +15,7 @@
See the License for the specific language governing permissions and
limitations under the License.import copy
"""
import colorsys
import io

View File

@ -93,14 +93,14 @@ class Plot:
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
if can_convert_to_int(row["result"]):
# value is not None
self.result_dict[model_name]["result"][
(int(row["batch_size"]), int(row["sequence_length"]))
] = int(row["result"])
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
int(row["result"])
)
elif can_convert_to_float(row["result"]):
# value is not None
self.result_dict[model_name]["result"][
(int(row["batch_size"]), int(row["sequence_length"]))
] = float(row["result"])
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
float(row["result"])
)
def plot(self):
fig, ax = plt.subplots()

View File

@ -15,6 +15,7 @@
"""
Post-processing utilities for question answering.
"""
import collections
import json
import logging

View File

@ -156,7 +156,7 @@ _deps = [
"rhoknp>=1.1.0,<1.3.1",
"rjieba",
"rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
"ruff==0.1.5",
"ruff==0.4.4",
"sacrebleu>=1.4.12,<2.0.0",
"sacremoses",
"safetensors>=0.4.1",

View File

@ -16,6 +16,7 @@
Audio processing functions to extract features from audio waveforms. This code is pure numpy to support all frameworks
and remove unnecessary dependencies.
"""
import warnings
from typing import Optional, Tuple, Union

View File

@ -17,7 +17,6 @@
Benchmarking the library on inference and training in PyTorch.
"""
import timeit
from typing import Callable, Optional

View File

@ -17,7 +17,6 @@
Benchmarking the library on inference and training in PyTorch.
"""
import random
import timeit
from functools import wraps

View File

@ -249,7 +249,6 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_i
else:
class MemoryMeasureProcess(Process):
"""
`MemoryMeasureProcess` inherits from `Process` and overwrites its `run()` method. Used to measure the
memory usage of a process

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""Configuration base class and utilities."""
import copy
import json
import os

View File

@ -400,7 +400,7 @@ def optimize(onnx_model_path: Path) -> Path:
sess_option.optimized_model_filepath = opt_model_path.as_posix()
_ = InferenceSession(onnx_model_path.as_posix(), sess_option)
print(f"Optimized model has been written at {opt_model_path}: \N{heavy check mark}")
print(f"Optimized model has been written at {opt_model_path}: \N{HEAVY CHECK MARK}")
print("/!\\ Optimized model contains hardware specific operators which might not be portable. /!\\")
return opt_model_path
@ -475,7 +475,7 @@ def quantize(onnx_model_path: Path) -> Path:
quantized_model_path = generate_identified_filename(onnx_model_path, "-quantized")
# Save model
print(f"Quantized model has been written at {quantized_model_path}: \N{heavy check mark}")
print(f"Quantized model has been written at {quantized_model_path}: \N{HEAVY CHECK MARK}")
onnx.save_model(quantizer.model.model, quantized_model_path.as_posix())
return quantized_model_path
@ -489,9 +489,9 @@ def verify(path: Path):
try:
onnx_options = SessionOptions()
_ = InferenceSession(path.as_posix(), onnx_options, providers=["CPUExecutionProvider"])
print(f"Model {path} correctly loaded: \N{heavy check mark}")
print(f"Model {path} correctly loaded: \N{HEAVY CHECK MARK}")
except RuntimeException as re:
print(f"Error while loading the model {re}: \N{heavy ballot x}")
print(f"Error while loading the model {re}: \N{HEAVY BALLOT X}")
if __name__ == "__main__":

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Convert pytorch checkpoints to TensorFlow"""
import argparse
import os

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Convert Seq2Seq TF Hub checkpoint."""
import argparse
from . import (

View File

@ -20,7 +20,6 @@ additional na_prob.json file is provided. This file is expected to map question
probability that a question is unanswerable.
"""
import collections
import json
import math

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""XNLI utils (dataset loading and evaluation)"""
import os
from ...utils import logging

View File

@ -17,6 +17,7 @@ in `integrations/deepspeed` instead.
Check: https://github.com/huggingface/transformers/pull/25599
"""
import warnings

View File

@ -62,7 +62,7 @@ deps = {
"rhoknp": "rhoknp>=1.1.0,<1.3.1",
"rjieba": "rjieba",
"rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
"ruff": "ruff==0.1.5",
"ruff": "ruff==0.4.4",
"sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
"sacremoses": "sacremoses",
"safetensors": "safetensors>=0.4.1",

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities to dynamically load objects from the Hub."""
import filecmp
import importlib
import importlib.util

View File

@ -15,6 +15,7 @@
"""
Sequence feature extraction class for common feature extractors to preprocess sequences.
"""
from typing import Dict, List, Optional, Union
import numpy as np

View File

@ -387,9 +387,9 @@ class StopStringCriteria(StoppingCriteria):
# Since this is lots of very small assignments of lists, we build it with numpy rather
# than torch for speed + simplicity, then convert to torch at the end
for token_idx, valid_positions in positions.items():
gather_vec[
token_idx, max_valid_positions * i : max_valid_positions * i + len(valid_positions)
] = valid_positions
gather_vec[token_idx, max_valid_positions * i : max_valid_positions * i + len(valid_positions)] = (
valid_positions
)
for token_idx, possible_end_lens in end_lens.items():
gather_vec[
token_idx,

View File

@ -68,7 +68,6 @@ class WatermarkDetectorOutput:
class WatermarkDetector:
r"""
Detector for detection of watermark generated text. The detector needs to be given the exact same settings that were
given during text generation to replicate the watermark greenlist generation and so detect the watermark. This includes

View File

@ -13,7 +13,6 @@
# limitations under the License.
"AQLM (Additive Quantization of Language Model) integration file"
from ..utils import is_accelerate_available, is_aqlm_available, is_torch_available

View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"AWQ (Activation aware Weight Quantization) integration file"
from ..activations import ACT2FN
from ..modeling_utils import PreTrainedModel
from ..utils import is_auto_awq_available, is_torch_available, logging

View File

@ -14,6 +14,7 @@
"""
Integration with Deepspeed
"""
import copy
import importlib.metadata as importlib_metadata
import importlib.util

View File

@ -17,6 +17,7 @@
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
"""
from array import array
import numpy as np

View File

@ -14,6 +14,7 @@
"""
Integrations with other Python libraries.
"""
import functools
import importlib.metadata
import importlib.util

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Configuration base class and utilities."""
import copy
import json
import os

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""PyTorch - Flax general utilities."""
import os
from pickle import UnpicklingError
from typing import Dict, Tuple

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""PyTorch - TF 2.0 general utilities."""
import os
import re

View File

@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""ALBERT model configuration"""
from collections import OrderedDict
from typing import Mapping

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Convert ALBERT checkpoint."""
import argparse
import torch

View File

@ -15,7 +15,6 @@
# limitations under the License.
"""TF 2.0 ALBERT model."""
from __future__ import annotations
import math

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Tokenization classes for ALBERT model."""
import os
import unicodedata
from shutil import copyfile

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Tokenization classes for ALBERT model."""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

View File

@ -16,7 +16,6 @@
Image/Text processor class for ALIGN
"""
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""AltCLIP model configuration"""
import os
from typing import Union

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""PyTorch AltCLIP model."""
import math
from dataclasses import dataclass
from typing import Any, List, Optional, Tuple, Union

View File

@ -15,6 +15,7 @@
"""
Image/Text processor class for AltCLIP
"""
import warnings
from ...processing_utils import ProcessorMixin

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Audio Spectogram Transformer (AST) model configuration"""
from ...configuration_utils import PretrainedConfig
from ...utils import logging

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Convert Audio Spectrogram Transformer checkpoints from the original repository. URL: https://github.com/YuanGongND/ast"""
import argparse
import json
from pathlib import Path

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Factory function to build auto-model classes."""
import copy
import importlib
import json

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Auto Config class."""
import importlib
import os
import re

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""AutoFeatureExtractor class."""
import importlib
import json
import os

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""AutoImageProcessor class."""
import importlib
import json
import os

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Auto Model class."""
from collections import OrderedDict
from ...utils import logging

View File

@ -14,7 +14,6 @@
# limitations under the License.
"""Auto Model class."""
import warnings
from collections import OrderedDict

Some files were not shown because too many files have changed in this diff Show More