mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-03 12:50:06 +06:00
update ruff version (#30932)
* update ruff version * fix research projects * Empty * Fix errors --------- Co-authored-by: Lysandre <lysandre@huggingface.co>
This commit is contained in:
parent
60bb571e99
commit
673440d073
2
Makefile
2
Makefile
@ -5,7 +5,7 @@ export PYTHONPATH = src
|
||||
|
||||
check_dirs := examples tests src utils
|
||||
|
||||
exclude_folders := examples/research_projects
|
||||
exclude_folders := ""
|
||||
|
||||
modified_only_fixup:
|
||||
$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))
|
||||
|
@ -20,6 +20,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -19,6 +19,7 @@ Pretraining the library models for T5-like span-masked language modeling on a te
|
||||
Here is the full list of checkpoints on the hub that can be pretrained by this script:
|
||||
https://huggingface.co/models?filter=t5
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Finetuning a 🤗 Flax Transformers model for sequence classification on GLUE."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Fine-tuning a 🤗 Flax Transformers model on token classification tasks (NER, POS, CHUNKS)"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -93,14 +93,14 @@ class Plot:
|
||||
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
|
||||
if can_convert_to_int(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = int(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
int(row["result"])
|
||||
)
|
||||
elif can_convert_to_float(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = float(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
float(row["result"])
|
||||
)
|
||||
|
||||
def plot(self):
|
||||
fig, ax = plt.subplots()
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Finetuning the library models for multiple choice (Bert, Roberta, XLNet)."""
|
||||
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension"""
|
||||
|
||||
|
||||
import csv
|
||||
import glob
|
||||
import json
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import logging
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Fine-tuning the library models for question-answering."""
|
||||
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -20,7 +20,6 @@ GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss.
|
||||
using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.
|
||||
"""
|
||||
|
||||
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
|
@ -28,6 +28,7 @@
|
||||
--output_dir ../log \
|
||||
--train_batch_size 16 \
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import logging
|
||||
|
@ -18,7 +18,6 @@
|
||||
Finetuning the library models for multiple choice on SWAG (Bert).
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import glob
|
||||
|
@ -21,7 +21,6 @@
|
||||
This script with default values evaluates a pretrained Transformer-XL on WikiText 103
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import math
|
||||
|
@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from argparse import REMAINDER, ArgumentParser
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Fine-tuning the library models for named entity recognition on CoNLL-2003."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Named entity recognition fine-tuning: utilities to work with CoNLL-2003 task."""
|
||||
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Finetuning any 🤗 Transformers model for image classification leveraging 🤗 Accelerate."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
A subclass of `Trainer` specific to Question-Answering tasks
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
A subclass of `Trainer` specific to Question-Answering tasks
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Finetuning a 🤗 Transformers model for sequence classification on GLUE."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
|
@ -14,9 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)
|
||||
"""
|
||||
|
||||
"""Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)"""
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
|
@ -19,7 +19,6 @@ Running this example:
|
||||
python run_generation_contrastive_search.py --model_name_or_path=openai-community/gpt2-large --penalty_alpha=0.6 --k=4 --length=256
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
|
@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from argparse import REMAINDER, ArgumentParser
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""PyTorch BERT model with Patience-based Early Exit."""
|
||||
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Training and inference using the library models for sequence classification on GLUE (Bert, Albert) with PABEE."""
|
||||
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""BertAbs configuration"""
|
||||
|
||||
import logging
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
@ -19,6 +19,7 @@
|
||||
Some parts of this script are adapted from the code of Michel et al. (http://arxiv.org/abs/1905.10650)
|
||||
which is available at https://github.com/pmichel31415/are-16-heads-really-better-than-1
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""The distiller to distil the student.
|
||||
Adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
@ -12,8 +12,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)
|
||||
"""
|
||||
"""Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)"""
|
||||
|
||||
import bisect
|
||||
import copy
|
||||
from collections import defaultdict
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""Dataset to distilled models
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import Dataset
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Preprocessing script before distillation.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pickle
|
||||
|
@ -16,6 +16,7 @@
|
||||
Preprocessing script before training the distilled model.
|
||||
Specific to RoBERTa -> DistilRoBERTa and GPT2 -> DistilGPT2.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
@ -16,6 +16,7 @@
|
||||
Preprocessing script before training DistilBERT.
|
||||
Specific to BERT -> DistilBERT.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Preprocessing script before training the distilled model.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pickle
|
||||
|
@ -16,6 +16,7 @@
|
||||
Training the distilled model.
|
||||
Supported architectures include: BERT -> DistilBERT, RoBERTa -> DistilRoBERTa, GPT2 -> DistilGPT2.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""Utils to train DistilBERT
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
@ -20,6 +20,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import colorsys
|
||||
import io
|
||||
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Finetuning the library models for multimodal multiclass prediction on MM-IMDB dataset."""
|
||||
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
|
@ -15,6 +15,7 @@
|
||||
Count remaining (non-zero) weights in the encoder (i.e. the transformer layers).
|
||||
Sparsity and remaining weights levels are equivalent: sparsity % = 100 - remaining weights %.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
@ -16,7 +16,6 @@
|
||||
"""Masked BERT model configuration. It replicates the class `~transformers.BertConfig`
|
||||
and adapts it to the specificities of MaskedBert (`pruning_method`, `mask_init` and `mask_scale`."""
|
||||
|
||||
|
||||
import logging
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
@ -18,7 +18,6 @@
|
||||
compute the adaptive mask.
|
||||
Built on top of `transformers.models.bert.modeling_bert`"""
|
||||
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Fine-pruning Masked BERT for question-answering on SQuAD."""
|
||||
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import logging
|
||||
|
@ -13,9 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
""" """
|
||||
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -19,6 +19,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Helper functions for training models with pytorch-quantization"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import colorsys
|
||||
import io
|
||||
|
||||
|
@ -93,14 +93,14 @@ class Plot:
|
||||
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
|
||||
if can_convert_to_int(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = int(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
int(row["result"])
|
||||
)
|
||||
elif can_convert_to_float(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = float(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
float(row["result"])
|
||||
)
|
||||
|
||||
def plot(self):
|
||||
fig, ax = plt.subplots()
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
2
setup.py
2
setup.py
@ -156,7 +156,7 @@ _deps = [
|
||||
"rhoknp>=1.1.0,<1.3.1",
|
||||
"rjieba",
|
||||
"rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
|
||||
"ruff==0.1.5",
|
||||
"ruff==0.4.4",
|
||||
"sacrebleu>=1.4.12,<2.0.0",
|
||||
"sacremoses",
|
||||
"safetensors>=0.4.1",
|
||||
|
@ -16,6 +16,7 @@
|
||||
Audio processing functions to extract features from audio waveforms. This code is pure numpy to support all frameworks
|
||||
and remove unnecessary dependencies.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
|
@ -17,7 +17,6 @@
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
"""
|
||||
|
||||
|
||||
import timeit
|
||||
from typing import Callable, Optional
|
||||
|
||||
|
@ -17,7 +17,6 @@
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
"""
|
||||
|
||||
|
||||
import random
|
||||
import timeit
|
||||
from functools import wraps
|
||||
|
@ -249,7 +249,6 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_i
|
||||
else:
|
||||
|
||||
class MemoryMeasureProcess(Process):
|
||||
|
||||
"""
|
||||
`MemoryMeasureProcess` inherits from `Process` and overwrites its `run()` method. Used to measure the
|
||||
memory usage of a process
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""Configuration base class and utilities."""
|
||||
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
|
@ -400,7 +400,7 @@ def optimize(onnx_model_path: Path) -> Path:
|
||||
sess_option.optimized_model_filepath = opt_model_path.as_posix()
|
||||
_ = InferenceSession(onnx_model_path.as_posix(), sess_option)
|
||||
|
||||
print(f"Optimized model has been written at {opt_model_path}: \N{heavy check mark}")
|
||||
print(f"Optimized model has been written at {opt_model_path}: \N{HEAVY CHECK MARK}")
|
||||
print("/!\\ Optimized model contains hardware specific operators which might not be portable. /!\\")
|
||||
|
||||
return opt_model_path
|
||||
@ -475,7 +475,7 @@ def quantize(onnx_model_path: Path) -> Path:
|
||||
quantized_model_path = generate_identified_filename(onnx_model_path, "-quantized")
|
||||
|
||||
# Save model
|
||||
print(f"Quantized model has been written at {quantized_model_path}: \N{heavy check mark}")
|
||||
print(f"Quantized model has been written at {quantized_model_path}: \N{HEAVY CHECK MARK}")
|
||||
onnx.save_model(quantizer.model.model, quantized_model_path.as_posix())
|
||||
|
||||
return quantized_model_path
|
||||
@ -489,9 +489,9 @@ def verify(path: Path):
|
||||
try:
|
||||
onnx_options = SessionOptions()
|
||||
_ = InferenceSession(path.as_posix(), onnx_options, providers=["CPUExecutionProvider"])
|
||||
print(f"Model {path} correctly loaded: \N{heavy check mark}")
|
||||
print(f"Model {path} correctly loaded: \N{HEAVY CHECK MARK}")
|
||||
except RuntimeException as re:
|
||||
print(f"Error while loading the model {re}: \N{heavy ballot x}")
|
||||
print(f"Error while loading the model {re}: \N{HEAVY BALLOT X}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Convert pytorch checkpoints to TensorFlow"""
|
||||
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Convert Seq2Seq TF Hub checkpoint."""
|
||||
|
||||
|
||||
import argparse
|
||||
|
||||
from . import (
|
||||
|
@ -20,7 +20,6 @@ additional na_prob.json file is provided. This file is expected to map question
|
||||
probability that a question is unanswerable.
|
||||
"""
|
||||
|
||||
|
||||
import collections
|
||||
import json
|
||||
import math
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""XNLI utils (dataset loading and evaluation)"""
|
||||
|
||||
|
||||
import os
|
||||
|
||||
from ...utils import logging
|
||||
|
@ -17,6 +17,7 @@ in `integrations/deepspeed` instead.
|
||||
|
||||
Check: https://github.com/huggingface/transformers/pull/25599
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
|
||||
|
@ -62,7 +62,7 @@ deps = {
|
||||
"rhoknp": "rhoknp>=1.1.0,<1.3.1",
|
||||
"rjieba": "rjieba",
|
||||
"rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
|
||||
"ruff": "ruff==0.1.5",
|
||||
"ruff": "ruff==0.4.4",
|
||||
"sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
|
||||
"sacremoses": "sacremoses",
|
||||
"safetensors": "safetensors>=0.4.1",
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Utilities to dynamically load objects from the Hub."""
|
||||
|
||||
import filecmp
|
||||
import importlib
|
||||
import importlib.util
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Sequence feature extraction class for common feature extractors to preprocess sequences.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
@ -387,9 +387,9 @@ class StopStringCriteria(StoppingCriteria):
|
||||
# Since this is lots of very small assignments of lists, we build it with numpy rather
|
||||
# than torch for speed + simplicity, then convert to torch at the end
|
||||
for token_idx, valid_positions in positions.items():
|
||||
gather_vec[
|
||||
token_idx, max_valid_positions * i : max_valid_positions * i + len(valid_positions)
|
||||
] = valid_positions
|
||||
gather_vec[token_idx, max_valid_positions * i : max_valid_positions * i + len(valid_positions)] = (
|
||||
valid_positions
|
||||
)
|
||||
for token_idx, possible_end_lens in end_lens.items():
|
||||
gather_vec[
|
||||
token_idx,
|
||||
|
@ -68,7 +68,6 @@ class WatermarkDetectorOutput:
|
||||
|
||||
|
||||
class WatermarkDetector:
|
||||
|
||||
r"""
|
||||
Detector for detection of watermark generated text. The detector needs to be given the exact same settings that were
|
||||
given during text generation to replicate the watermark greenlist generation and so detect the watermark. This includes
|
||||
|
@ -13,7 +13,6 @@
|
||||
# limitations under the License.
|
||||
"AQLM (Additive Quantization of Language Model) integration file"
|
||||
|
||||
|
||||
from ..utils import is_accelerate_available, is_aqlm_available, is_torch_available
|
||||
|
||||
|
||||
|
@ -12,6 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"AWQ (Activation aware Weight Quantization) integration file"
|
||||
|
||||
from ..activations import ACT2FN
|
||||
from ..modeling_utils import PreTrainedModel
|
||||
from ..utils import is_auto_awq_available, is_torch_available, logging
|
||||
|
@ -14,6 +14,7 @@
|
||||
"""
|
||||
Integration with Deepspeed
|
||||
"""
|
||||
|
||||
import copy
|
||||
import importlib.metadata as importlib_metadata
|
||||
import importlib.util
|
||||
|
@ -17,6 +17,7 @@
|
||||
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
|
||||
with extra methods beings exposed
|
||||
"""
|
||||
|
||||
from array import array
|
||||
|
||||
import numpy as np
|
||||
|
@ -14,6 +14,7 @@
|
||||
"""
|
||||
Integrations with other Python libraries.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Configuration base class and utilities."""
|
||||
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""PyTorch - Flax general utilities."""
|
||||
|
||||
|
||||
import os
|
||||
from pickle import UnpicklingError
|
||||
from typing import Dict, Tuple
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""PyTorch - TF 2.0 general utilities."""
|
||||
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
|
@ -14,6 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""ALBERT model configuration"""
|
||||
|
||||
from collections import OrderedDict
|
||||
from typing import Mapping
|
||||
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Convert ALBERT checkpoint."""
|
||||
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
@ -15,7 +15,6 @@
|
||||
# limitations under the License.
|
||||
"""TF 2.0 ALBERT model."""
|
||||
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for ALBERT model."""
|
||||
|
||||
|
||||
import os
|
||||
import unicodedata
|
||||
from shutil import copyfile
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Tokenization classes for ALBERT model."""
|
||||
|
||||
|
||||
import os
|
||||
from shutil import copyfile
|
||||
from typing import List, Optional, Tuple
|
||||
|
@ -16,7 +16,6 @@
|
||||
Image/Text processor class for ALIGN
|
||||
"""
|
||||
|
||||
|
||||
from ...processing_utils import ProcessorMixin
|
||||
from ...tokenization_utils_base import BatchEncoding
|
||||
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""AltCLIP model configuration"""
|
||||
|
||||
import os
|
||||
from typing import Union
|
||||
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""PyTorch AltCLIP model."""
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, List, Optional, Tuple, Union
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Image/Text processor class for AltCLIP
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
from ...processing_utils import ProcessorMixin
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Audio Spectogram Transformer (AST) model configuration"""
|
||||
|
||||
|
||||
from ...configuration_utils import PretrainedConfig
|
||||
from ...utils import logging
|
||||
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Convert Audio Spectrogram Transformer checkpoints from the original repository. URL: https://github.com/YuanGongND/ast"""
|
||||
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Factory function to build auto-model classes."""
|
||||
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Auto Config class."""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
import re
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""AutoFeatureExtractor class."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""AutoImageProcessor class."""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import os
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Auto Model class."""
|
||||
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from ...utils import logging
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Auto Model class."""
|
||||
|
||||
|
||||
import warnings
|
||||
from collections import OrderedDict
|
||||
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user