mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-03 12:50:06 +06:00
update ruff version (#30932)
* update ruff version * fix research projects * Empty * Fix errors --------- Co-authored-by: Lysandre <lysandre@huggingface.co>
This commit is contained in:
parent
60bb571e99
commit
673440d073
2
Makefile
2
Makefile
@ -5,7 +5,7 @@ export PYTHONPATH = src
|
||||
|
||||
check_dirs := examples tests src utils
|
||||
|
||||
exclude_folders := examples/research_projects
|
||||
exclude_folders := ""
|
||||
|
||||
modified_only_fixup:
|
||||
$(eval modified_py_files := $(shell python utils/get_modified_files.py $(check_dirs)))
|
||||
|
@ -20,6 +20,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -19,6 +19,7 @@ Pretraining the library models for T5-like span-masked language modeling on a te
|
||||
Here is the full list of checkpoints on the hub that can be pretrained by this script:
|
||||
https://huggingface.co/models?filter=t5
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -13,7 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning a 🤗 Flax Transformers model for sequence classification on GLUE."""
|
||||
"""Finetuning a 🤗 Flax Transformers model for sequence classification on GLUE."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -13,7 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-tuning a 🤗 Flax Transformers model on token classification tasks (NER, POS, CHUNKS)"""
|
||||
"""Fine-tuning a 🤗 Flax Transformers model on token classification tasks (NER, POS, CHUNKS)"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
|
@ -93,14 +93,14 @@ class Plot:
|
||||
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
|
||||
if can_convert_to_int(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = int(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
int(row["result"])
|
||||
)
|
||||
elif can_convert_to_float(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = float(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
float(row["result"])
|
||||
)
|
||||
|
||||
def plot(self):
|
||||
fig, ax = plt.subplots()
|
||||
|
@ -14,7 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Benchmarking the library on inference and training """
|
||||
"""Benchmarking the library on inference and training"""
|
||||
|
||||
from transformers import HfArgumentParser, PyTorchBenchmark, PyTorchBenchmarkArguments
|
||||
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for multiple choice (Bert, Roberta, XLNet)."""
|
||||
|
||||
"""Finetuning the library models for multiple choice (Bert, Roberta, XLNet)."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension """
|
||||
|
||||
"""Multiple choice fine-tuning: utilities to work with multiple choice tasks of reading comprehension"""
|
||||
|
||||
import csv
|
||||
import glob
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
|
||||
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-tuning the library models for question-answering."""
|
||||
|
||||
"""Fine-tuning the library models for question-answering."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -20,7 +20,6 @@ GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss.
|
||||
using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.
|
||||
"""
|
||||
|
||||
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
|
@ -28,6 +28,7 @@
|
||||
--output_dir ../log \
|
||||
--train_batch_size 16 \
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import logging
|
||||
|
@ -15,10 +15,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""BERT finetuning runner.
|
||||
Finetuning the library models for multiple choice on SWAG (Bert).
|
||||
Finetuning the library models for multiple choice on SWAG (Bert).
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import glob
|
||||
|
@ -14,14 +14,13 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" PyTorch Transformer XL model evaluation script.
|
||||
Adapted from https://github.com/kimiyoung/transformer-xl.
|
||||
In particular https://github.com/kimiyoung/transformer-xl/blob/master/pytorch/eval.py
|
||||
"""PyTorch Transformer XL model evaluation script.
|
||||
Adapted from https://github.com/kimiyoung/transformer-xl.
|
||||
In particular https://github.com/kimiyoung/transformer-xl/blob/master/pytorch/eval.py
|
||||
|
||||
This script with default values evaluates a pretrained Transformer-XL on WikiText 103
|
||||
This script with default values evaluates a pretrained Transformer-XL on WikiText 103
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import math
|
||||
|
@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from argparse import REMAINDER, ArgumentParser
|
||||
|
@ -13,7 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-tuning the library models for named entity recognition on CoNLL-2003. """
|
||||
"""Fine-tuning the library models for named entity recognition on CoNLL-2003."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Named entity recognition fine-tuning: utilities to work with CoNLL-2003 task. """
|
||||
|
||||
"""Named entity recognition fine-tuning: utilities to work with CoNLL-2003 task."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -12,7 +12,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning any 🤗 Transformers model for image classification leveraging 🤗 Accelerate."""
|
||||
"""Finetuning any 🤗 Transformers model for image classification leveraging 🤗 Accelerate."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
|
@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning 🤗 Transformers model for object detection with Accelerate."""
|
||||
"""Finetuning 🤗 Transformers model for object detection with Accelerate."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
A subclass of `Trainer` specific to Question-Answering tasks
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
A subclass of `Trainer` specific to Question-Answering tasks
|
||||
"""
|
||||
|
||||
import math
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning any 🤗 Transformers model supported by AutoModelForSemanticSegmentation for semantic segmentation."""
|
||||
"""Finetuning any 🤗 Transformers model supported by AutoModelForSemanticSegmentation for semantic segmentation."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
|
||||
""" Pre-Training a 🤗 Wav2Vec2 model on unlabeled audio data """
|
||||
"""Pre-Training a 🤗 Wav2Vec2 model on unlabeled audio data"""
|
||||
|
||||
import argparse
|
||||
import math
|
||||
|
@ -14,7 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
|
||||
"""Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
@ -14,7 +14,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
""" Fine-tuning a 🤗 Transformers CTC adapter model for automatic speech recognition"""
|
||||
"""Fine-tuning a 🤗 Transformers CTC adapter model for automatic speech recognition"""
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for text classification."""
|
||||
"""Finetuning the library models for text classification."""
|
||||
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
|
||||
|
||||
import logging
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for sequence classification on GLUE."""
|
||||
"""Finetuning the library models for sequence classification on GLUE."""
|
||||
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
|
||||
|
||||
import logging
|
||||
|
@ -12,7 +12,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning a 🤗 Transformers model for sequence classification on GLUE."""
|
||||
"""Finetuning a 🤗 Transformers model for sequence classification on GLUE."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
|
@ -14,8 +14,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning multi-lingual models on XNLI (e.g. Bert, DistilBERT, XLM).
|
||||
Adapted from `examples/text-classification/run_glue.py`"""
|
||||
"""Finetuning multi-lingual models on XNLI (e.g. Bert, DistilBERT, XLM).
|
||||
Adapted from `examples/text-classification/run_glue.py`"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -14,9 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)
|
||||
"""
|
||||
|
||||
"""Conditional text generation with the auto-regressive models of the library (GPT/GPT-2/CTRL/Transformer-XL/XLNet)"""
|
||||
|
||||
import argparse
|
||||
import inspect
|
||||
|
@ -13,13 +13,12 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" The examples of running contrastive search on the auto-APIs;
|
||||
"""The examples of running contrastive search on the auto-APIs;
|
||||
|
||||
Running this example:
|
||||
python run_generation_contrastive_search.py --model_name_or_path=openai-community/gpt2-large --penalty_alpha=0.6 --k=4 --length=256
|
||||
"""
|
||||
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
|
||||
|
@ -23,7 +23,6 @@ Inspired by https://github.com/pytorch/pytorch/blob/master/torch/distributed/lau
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
from argparse import REMAINDER, ArgumentParser
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for sequence classification on HANS."""
|
||||
"""Finetuning the library models for sequence classification on HANS."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""PyTorch ALBERT model with Patience-based Early Exit. """
|
||||
"""PyTorch ALBERT model with Patience-based Early Exit."""
|
||||
|
||||
import logging
|
||||
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""PyTorch BERT model with Patience-based Early Exit. """
|
||||
|
||||
"""PyTorch BERT model with Patience-based Early Exit."""
|
||||
|
||||
import logging
|
||||
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Training and inference using the library models for sequence classification on GLUE (Bert, Albert) with PABEE."""
|
||||
|
||||
"""Training and inference using the library models for sequence classification on GLUE (Bert, Albert) with PABEE."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -13,7 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" BertAbs configuration """
|
||||
"""BertAbs configuration"""
|
||||
|
||||
import logging
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Convert BertExtAbs's checkpoints.
|
||||
"""Convert BertExtAbs's checkpoints.
|
||||
|
||||
The script looks like it is doing something trivial but it is not. The "weights"
|
||||
proposed by the authors are actually the entire model pickled. We need to load
|
||||
|
@ -12,13 +12,14 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Bertology: this script shows how you can explore the internals of the models in the library to:
|
||||
- compute the entropy of the head attentions
|
||||
- compute the importance of each head
|
||||
- prune (remove) the low importance head.
|
||||
Some parts of this script are adapted from the code of Michel et al. (http://arxiv.org/abs/1905.10650)
|
||||
which is available at https://github.com/pmichel31415/are-16-heads-really-better-than-1
|
||||
"""Bertology: this script shows how you can explore the internals of the models in the library to:
|
||||
- compute the entropy of the head attentions
|
||||
- compute the importance of each head
|
||||
- prune (remove) the low importance head.
|
||||
Some parts of this script are adapted from the code of Michel et al. (http://arxiv.org/abs/1905.10650)
|
||||
which is available at https://github.com/pmichel31415/are-16-heads-really-better-than-1
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
""" This script is adapted from the Bertology pruning code (https://github.com/huggingface/transformers/blob/783d7d2629e97c5f0c5f9ef01b8c66410275c204/examples/research_projects/bertology/run_bertology.py)
|
||||
"""This script is adapted from the Bertology pruning code (https://github.com/huggingface/transformers/blob/783d7d2629e97c5f0c5f9ef01b8c66410275c204/examples/research_projects/bertology/run_bertology.py)
|
||||
to prune GPT-like models. The author is @altsoph.
|
||||
"""
|
||||
|
||||
|
@ -12,9 +12,10 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" The distiller to distil the student.
|
||||
Adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""The distiller to distil the student.
|
||||
Adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
|
@ -12,8 +12,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)
|
||||
"""
|
||||
"""Adapted from PyTorch Vision (https://github.com/pytorch/vision/blob/master/references/detection/group_by_aspect_ratio.py)"""
|
||||
|
||||
import bisect
|
||||
import copy
|
||||
from collections import defaultdict
|
||||
|
@ -12,9 +12,10 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Dataset to distilled models
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""Dataset to distilled models
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch.utils.data import Dataset
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" This is the exact same script as `examples/question-answering/run_squad.py` (as of 2020, January 8th) with an additional and optional step of distillation."""
|
||||
"""This is the exact same script as `examples/question-answering/run_squad.py` (as of 2020, January 8th) with an additional and optional step of distillation."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Preprocessing script before distillation.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pickle
|
||||
|
@ -16,6 +16,7 @@
|
||||
Preprocessing script before training the distilled model.
|
||||
Specific to RoBERTa -> DistilRoBERTa and GPT2 -> DistilGPT2.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
@ -16,6 +16,7 @@
|
||||
Preprocessing script before training DistilBERT.
|
||||
Specific to BERT -> DistilBERT.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import torch
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Preprocessing script before training the distilled model.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pickle
|
||||
|
@ -16,6 +16,7 @@
|
||||
Training the distilled model.
|
||||
Supported architectures include: BERT -> DistilBERT, RoBERTa -> DistilRoBERTa, GPT2 -> DistilGPT2.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
@ -12,9 +12,10 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Utils to train DistilBERT
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""Utils to train DistilBERT
|
||||
adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
@ -20,6 +20,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2 && Huggingface Co.
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2 && Huggingface Co.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
|
@ -1,20 +1,20 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal, Huggingface team :)
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal, Huggingface team :)
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import copy
|
||||
import fnmatch
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import colorsys
|
||||
import io
|
||||
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for multimodal multiclass prediction on MM-IMDB dataset."""
|
||||
|
||||
"""Finetuning the library models for multimodal multiclass prediction on MM-IMDB dataset."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -15,6 +15,7 @@
|
||||
Count remaining (non-zero) weights in the encoder (i.e. the transformer layers).
|
||||
Sparsity and remaining weights levels are equivalent: sparsity % = 100 - remaining weights %.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
|
@ -13,10 +13,9 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Masked BERT model configuration. It replicates the class `~transformers.BertConfig`
|
||||
"""Masked BERT model configuration. It replicates the class `~transformers.BertConfig`
|
||||
and adapts it to the specificities of MaskedBert (`pruning_method`, `mask_init` and `mask_scale`."""
|
||||
|
||||
|
||||
import logging
|
||||
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
@ -18,7 +18,6 @@
|
||||
compute the adaptive mask.
|
||||
Built on top of `transformers.models.bert.modeling_bert`"""
|
||||
|
||||
|
||||
import logging
|
||||
import math
|
||||
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-pruning Masked BERT on sequence classification on GLUE."""
|
||||
"""Fine-pruning Masked BERT on sequence classification on GLUE."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-pruning Masked BERT for question-answering on SQuAD."""
|
||||
|
||||
"""Fine-pruning Masked BERT for question-answering on SQuAD."""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
|
@ -13,9 +13,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
""" """
|
||||
|
||||
"""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -19,6 +19,7 @@ text file or a dataset.
|
||||
Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
|
||||
https://huggingface.co/models?filter=fill-mask
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
@ -12,7 +12,8 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
"""Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet)."""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Helper functions for training models with pytorch-quantization"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -1,4 +1,4 @@
|
||||
""" Evaluation script for RAG models."""
|
||||
"""Evaluation script for RAG models."""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
|
@ -1,4 +1,4 @@
|
||||
""" Evaluation script for RAG models."""
|
||||
"""Evaluation script for RAG models."""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
|
||||
""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
|
||||
"""Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition"""
|
||||
|
||||
import functools
|
||||
import json
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
|
||||
""" Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition in streaming mode"""
|
||||
"""Fine-tuning a 🤗 Transformers CTC model for automatic speech recognition in streaming mode"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2 && Huggingface Co.
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2 && Huggingface Co.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import itertools
|
||||
import math
|
||||
import os
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
|
@ -1,20 +1,20 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal, Huggingface team :)
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal, Huggingface team :)
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
import copy
|
||||
import fnmatch
|
||||
|
@ -1,20 +1,21 @@
|
||||
"""
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
coding=utf-8
|
||||
Copyright 2018, Antonio Mendoza Hao Tan, Mohit Bansal
|
||||
Adapted From Facebook Inc, Detectron2
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.import copy
|
||||
"""
|
||||
import colorsys
|
||||
import io
|
||||
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
|
||||
""" Fine-tuning a 🤗 Transformers pretrained speech model on the XTREME-S benchmark tasks"""
|
||||
"""Fine-tuning a 🤗 Transformers pretrained speech model on the XTREME-S benchmark tasks"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
@ -93,14 +93,14 @@ class Plot:
|
||||
self.result_dict[model_name]["seq_len"].append(int(row["sequence_length"]))
|
||||
if can_convert_to_int(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = int(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
int(row["result"])
|
||||
)
|
||||
elif can_convert_to_float(row["result"]):
|
||||
# value is not None
|
||||
self.result_dict[model_name]["result"][
|
||||
(int(row["batch_size"]), int(row["sequence_length"]))
|
||||
] = float(row["result"])
|
||||
self.result_dict[model_name]["result"][(int(row["batch_size"]), int(row["sequence_length"]))] = (
|
||||
float(row["result"])
|
||||
)
|
||||
|
||||
def plot(self):
|
||||
fig, ax = plt.subplots()
|
||||
|
@ -14,7 +14,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Benchmarking the library on inference and training in TensorFlow"""
|
||||
"""Benchmarking the library on inference and training in TensorFlow"""
|
||||
|
||||
from transformers import HfArgumentParser, TensorFlowBenchmark, TensorFlowBenchmarkArguments
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
"""
|
||||
Post-processing utilities for question answering.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import json
|
||||
import logging
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Finetuning the library models for sequence classification on GLUE."""
|
||||
"""Finetuning the library models for sequence classification on GLUE."""
|
||||
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
|
||||
|
||||
import json
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Fine-tuning the library models for sequence classification."""
|
||||
"""Fine-tuning the library models for sequence classification."""
|
||||
# You can also adapt this script on your own text classification task. Pointers for this are left as comments.
|
||||
|
||||
import json
|
||||
|
2
setup.py
2
setup.py
@ -156,7 +156,7 @@ _deps = [
|
||||
"rhoknp>=1.1.0,<1.3.1",
|
||||
"rjieba",
|
||||
"rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
|
||||
"ruff==0.1.5",
|
||||
"ruff==0.4.4",
|
||||
"sacrebleu>=1.4.12,<2.0.0",
|
||||
"sacremoses",
|
||||
"safetensors>=0.4.1",
|
||||
|
@ -16,6 +16,7 @@
|
||||
Audio processing functions to extract features from audio waveforms. This code is pure numpy to support all frameworks
|
||||
and remove unnecessary dependencies.
|
||||
"""
|
||||
|
||||
import warnings
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
|
@ -14,10 +14,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
"""
|
||||
|
||||
|
||||
import timeit
|
||||
from typing import Callable, Optional
|
||||
|
||||
|
@ -14,10 +14,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
Benchmarking the library on inference and training in PyTorch.
|
||||
"""
|
||||
|
||||
|
||||
import random
|
||||
import timeit
|
||||
from functools import wraps
|
||||
|
@ -249,7 +249,6 @@ def measure_peak_memory_cpu(function: Callable[[], None], interval=0.5, device_i
|
||||
else:
|
||||
|
||||
class MemoryMeasureProcess(Process):
|
||||
|
||||
"""
|
||||
`MemoryMeasureProcess` inherits from `Process` and overwrites its `run()` method. Used to measure the
|
||||
memory usage of a process
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Configuration base class and utilities."""
|
||||
|
||||
"""Configuration base class and utilities."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
@ -400,7 +400,7 @@ def optimize(onnx_model_path: Path) -> Path:
|
||||
sess_option.optimized_model_filepath = opt_model_path.as_posix()
|
||||
_ = InferenceSession(onnx_model_path.as_posix(), sess_option)
|
||||
|
||||
print(f"Optimized model has been written at {opt_model_path}: \N{heavy check mark}")
|
||||
print(f"Optimized model has been written at {opt_model_path}: \N{HEAVY CHECK MARK}")
|
||||
print("/!\\ Optimized model contains hardware specific operators which might not be portable. /!\\")
|
||||
|
||||
return opt_model_path
|
||||
@ -475,7 +475,7 @@ def quantize(onnx_model_path: Path) -> Path:
|
||||
quantized_model_path = generate_identified_filename(onnx_model_path, "-quantized")
|
||||
|
||||
# Save model
|
||||
print(f"Quantized model has been written at {quantized_model_path}: \N{heavy check mark}")
|
||||
print(f"Quantized model has been written at {quantized_model_path}: \N{HEAVY CHECK MARK}")
|
||||
onnx.save_model(quantizer.model.model, quantized_model_path.as_posix())
|
||||
|
||||
return quantized_model_path
|
||||
@ -489,9 +489,9 @@ def verify(path: Path):
|
||||
try:
|
||||
onnx_options = SessionOptions()
|
||||
_ = InferenceSession(path.as_posix(), onnx_options, providers=["CPUExecutionProvider"])
|
||||
print(f"Model {path} correctly loaded: \N{heavy check mark}")
|
||||
print(f"Model {path} correctly loaded: \N{HEAVY CHECK MARK}")
|
||||
except RuntimeException as re:
|
||||
print(f"Error while loading the model {re}: \N{heavy ballot x}")
|
||||
print(f"Error while loading the model {re}: \N{HEAVY BALLOT X}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -12,8 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Convert pytorch checkpoints to TensorFlow"""
|
||||
|
||||
"""Convert pytorch checkpoints to TensorFlow"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
@ -12,7 +12,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library)"""
|
||||
"""Convert slow tokenizers checkpoints in fast (serialization format of the `tokenizers` library)"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
@ -14,7 +14,6 @@
|
||||
# limitations under the License.
|
||||
"""Convert Seq2Seq TF Hub checkpoint."""
|
||||
|
||||
|
||||
import argparse
|
||||
|
||||
from . import (
|
||||
|
@ -20,7 +20,6 @@ additional na_prob.json file is provided. This file is expected to map question
|
||||
probability that a question is unanswerable.
|
||||
"""
|
||||
|
||||
|
||||
import collections
|
||||
import json
|
||||
import math
|
||||
|
@ -13,7 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" GLUE processors and helpers"""
|
||||
"""GLUE processors and helpers"""
|
||||
|
||||
import os
|
||||
import warnings
|
||||
|
@ -13,8 +13,7 @@
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
""" XNLI utils (dataset loading and evaluation)"""
|
||||
|
||||
"""XNLI utils (dataset loading and evaluation)"""
|
||||
|
||||
import os
|
||||
|
||||
|
@ -17,6 +17,7 @@ in `integrations/deepspeed` instead.
|
||||
|
||||
Check: https://github.com/huggingface/transformers/pull/25599
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
|
||||
|
@ -62,7 +62,7 @@ deps = {
|
||||
"rhoknp": "rhoknp>=1.1.0,<1.3.1",
|
||||
"rjieba": "rjieba",
|
||||
"rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
|
||||
"ruff": "ruff==0.1.5",
|
||||
"ruff": "ruff==0.4.4",
|
||||
"sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
|
||||
"sacremoses": "sacremoses",
|
||||
"safetensors": "safetensors>=0.4.1",
|
||||
|
@ -13,6 +13,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Utilities to dynamically load objects from the Hub."""
|
||||
|
||||
import filecmp
|
||||
import importlib
|
||||
import importlib.util
|
||||
|
@ -13,8 +13,9 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Sequence feature extraction class for common feature extractors to preprocess sequences.
|
||||
Sequence feature extraction class for common feature extractors to preprocess sequences.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user