mirror of
https://github.com/huggingface/transformers.git
synced 2025-07-31 10:12:23 +06:00
Remove deprecated get_cached_models
(#35809)
* Remove deprecated get_cached_models * imports
This commit is contained in:
parent
97fbaf0861
commit
7051c5fcc8
@ -71,7 +71,6 @@ from .utils import (
|
||||
copy_func,
|
||||
default_cache_path,
|
||||
define_sagemaker_information,
|
||||
get_cached_models,
|
||||
get_file_from_repo,
|
||||
get_torch_version,
|
||||
has_file,
|
||||
|
@ -91,7 +91,6 @@ from .hub import (
|
||||
define_sagemaker_information,
|
||||
download_url,
|
||||
extract_commit_hash,
|
||||
get_cached_models,
|
||||
get_file_from_repo,
|
||||
has_file,
|
||||
http_user_agent,
|
||||
|
@ -25,7 +25,7 @@ import traceback
|
||||
import warnings
|
||||
from concurrent import futures
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Dict, List, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
@ -60,7 +60,6 @@ from huggingface_hub.utils import (
|
||||
hf_raise_for_status,
|
||||
send_telemetry,
|
||||
)
|
||||
from huggingface_hub.utils._deprecation import _deprecate_method
|
||||
from requests.exceptions import HTTPError
|
||||
|
||||
from . import __version__, logging
|
||||
@ -165,45 +164,6 @@ def is_remote_url(url_or_filename):
|
||||
return parsed.scheme in ("http", "https")
|
||||
|
||||
|
||||
# TODO: remove this once fully deprecated
|
||||
# TODO? remove from './examples/research_projects/lxmert/utils.py' as well
|
||||
# TODO? remove from './examples/research_projects/visual_bert/utils.py' as well
|
||||
@_deprecate_method(version="4.39.0", message="This method is outdated and does not support the new cache system.")
|
||||
def get_cached_models(cache_dir: Union[str, Path] = None) -> List[Tuple]:
|
||||
"""
|
||||
Returns a list of tuples representing model binaries that are cached locally. Each tuple has shape `(model_url,
|
||||
etag, size_MB)`. Filenames in `cache_dir` are use to get the metadata for each model, only urls ending with *.bin*
|
||||
are added.
|
||||
|
||||
Args:
|
||||
cache_dir (`Union[str, Path]`, *optional*):
|
||||
The cache directory to search for models within. Will default to the transformers cache if unset.
|
||||
|
||||
Returns:
|
||||
List[Tuple]: List of tuples each with shape `(model_url, etag, size_MB)`
|
||||
"""
|
||||
if cache_dir is None:
|
||||
cache_dir = TRANSFORMERS_CACHE
|
||||
elif isinstance(cache_dir, Path):
|
||||
cache_dir = str(cache_dir)
|
||||
if not os.path.isdir(cache_dir):
|
||||
return []
|
||||
|
||||
cached_models = []
|
||||
for file in os.listdir(cache_dir):
|
||||
if file.endswith(".json"):
|
||||
meta_path = os.path.join(cache_dir, file)
|
||||
with open(meta_path, encoding="utf-8") as meta_file:
|
||||
metadata = json.load(meta_file)
|
||||
url = metadata["url"]
|
||||
etag = metadata["etag"]
|
||||
if url.endswith(".bin"):
|
||||
size_MB = os.path.getsize(meta_path.strip(".json")) / 1e6
|
||||
cached_models.append((url, etag, size_MB))
|
||||
|
||||
return cached_models
|
||||
|
||||
|
||||
def define_sagemaker_information():
|
||||
try:
|
||||
instance_data = requests.get(os.environ["ECS_CONTAINER_METADATA_URI"]).json()
|
||||
|
Loading…
Reference in New Issue
Block a user