From 813d730c4699a6f26cd7a6bf2e6ea37e815d09c4 Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Tue, 16 Mar 2021 08:41:47 -0400 Subject: [PATCH] Release utils (#10735) * Examples version update * Refactor a bit * All version updates * Fixes * README cleanup * Post-release/patch * Fixes * More fixes * Tests * More fixes * Moar fixes * Make commands and update setup * Replace spaces with weird tabs * Fix test * Style --- Makefile | 15 +++ setup.py | 24 ++--- utils/release.py | 256 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 280 insertions(+), 15 deletions(-) create mode 100644 utils/release.py diff --git a/Makefile b/Makefile index c3ac1df6239..7974335c14d 100644 --- a/Makefile +++ b/Makefile @@ -69,3 +69,18 @@ test-examples: docs: cd docs && make html SPHINXOPTS="-W -j 4" + +# Release stuff + +pre-release: + python utils/release.py + +pre-patch: + python utils/release.py --patch + +post-release: + python utils/release.py --post_release + +post-patch: + python utils/release.py --post_release --patch + diff --git a/setup.py b/setup.py index c27c66ff8cd..261e90f1125 100644 --- a/setup.py +++ b/setup.py @@ -17,19 +17,17 @@ Simple check list from AllenNLP repo: https://github.com/allenai/allennlp/blob/m To create the package for pypi. -1. Change the version in __init__.py, setup.py as well as docs/source/conf.py. Remove the master from the links in - the new models of the README: - (https://huggingface.co/transformers/master/model_doc/ -> https://huggingface.co/transformers/model_doc/) - then run `make fix-copies` to fix the index of the documentation. +1. Run `make pre-release` (or `make pre-patch` for a patch release) then run `make fix-copies` to fix the index of the + documentation. 2. Unpin specific versions from setup.py that use a git install. -2. Commit these changes with the message: "Release: VERSION" +3. Commit these changes with the message: "Release: VERSION" -3. Add a tag in git to mark the release: "git tag VERSION -m 'Adds tag VERSION for pypi' " +4. Add a tag in git to mark the release: "git tag VERSION -m 'Adds tag VERSION for pypi' " Push the tag to git: git push --tags origin master -4. Build both the sources and the wheel. Do not change anything in setup.py between +5. Build both the sources and the wheel. Do not change anything in setup.py between creating the wheel and the source distribution (obviously). For the wheel, run: "python setup.py bdist_wheel" in the top level directory. @@ -38,7 +36,7 @@ To create the package for pypi. For the sources, run: "python setup.py sdist" You should now have a /dist directory with both .whl and .tar.gz source versions. -5. Check that everything looks correct by uploading the package to the pypi test server: +6. Check that everything looks correct by uploading the package to the pypi test server: twine upload dist/* -r pypitest (pypi suggest using twine as other methods upload files via plaintext.) @@ -48,16 +46,12 @@ To create the package for pypi. Check that you can install it in a virtualenv by running: pip install -i https://testpypi.python.org/pypi transformers -6. Upload the final version to actual pypi: +7. Upload the final version to actual pypi: twine upload dist/* -r pypi -7. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory. +8. Copy the release notes from RELEASE.md to the tag in github once everything is looking hunky-dory. -8. Add the release version to docs/source/_static/js/custom.js and .circleci/deploy.sh - -9. Update README.md to redirect to correct documentation. - -10. Update the version in __init__.py, setup.py to the new version "-dev" and push to master. +9. Run `make post-release` (or `make post-patch` for a patch release). """ import os diff --git a/utils/release.py b/utils/release.py new file mode 100644 index 00000000000..9fea1ab8406 --- /dev/null +++ b/utils/release.py @@ -0,0 +1,256 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re + +import git +import packaging.version + + +PATH_TO_EXAMPLES = "examples/" +REPLACE_PATTERNS = { + "examples": (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'), + "init": (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'), + "setup": (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'), + "doc": (re.compile(r"^(\s*)release\s*=\s*u'[^']+'$", re.MULTILINE), "release = u'VERSION'\n"), +} +REPLACE_FILES = { + "init": "src/transformers/__init__.py", + "setup": "setup.py", + "doc": "docs/source/conf.py", +} +README_FILE = "README.md" +CUSTOM_JS_FILE = "docs/source/_static/js/custom.js" +DEPLOY_SH_FILE = ".circleci/deploy.sh" + + +def update_version_in_file(fname, version, pattern): + """Update the version in one file using a specific pattern.""" + with open(fname, "r", encoding="utf-8", newline="\n") as f: + code = f.read() + re_pattern, replace = REPLACE_PATTERNS[pattern] + replace = replace.replace("VERSION", version) + code = re_pattern.sub(replace, code) + with open(fname, "w", encoding="utf-8", newline="\n") as f: + f.write(code) + + +def update_version_in_examples(version): + """Update the version in all examples files.""" + for folder, directories, fnames in os.walk(PATH_TO_EXAMPLES): + # Removing some of the folders with non-actively maintained examples from the walk + if "research_projects" in directories: + directories.remove("research_projects") + if "legacy" in directories: + directories.remove("legacy") + for fname in fnames: + if fname.endswith(".py"): + update_version_in_file(os.path.join(folder, fname), version, pattern="examples") + + +def global_version_update(version, patch=False): + """Update the version in all needed files.""" + for pattern, fname in REPLACE_FILES.items(): + update_version_in_file(fname, version, pattern) + if not patch: + update_version_in_examples(version) + + +def clean_master_ref_in_model_list(): + """Replace the links from master doc tp stable doc in the model list of the README.""" + # If the introduction or the conclusion of the list change, the prompts may need to be updated. + _start_prompt = "🤗 Transformers currently provides the following architectures" + _end_prompt = "1. Want to contribute a new model?" + with open(README_FILE, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + + # Find the start of the list. + start_index = 0 + while not lines[start_index].startswith(_start_prompt): + start_index += 1 + start_index += 1 + + index = start_index + # Update the lines in the model list. + while not lines[index].startswith(_end_prompt): + if lines[index].startswith("1."): + lines[index] = lines[index].replace( + "https://huggingface.co/transformers/master/model_doc", + "https://huggingface.co/transformers/model_doc", + ) + index += 1 + + with open(README_FILE, "w", encoding="utf-8", newline="\n") as f: + f.writelines(lines) + + +def get_version(): + """Reads the current version in the __init__.""" + with open(REPLACE_FILES["init"], "r") as f: + code = f.read() + default_version = REPLACE_PATTERNS["init"][0].search(code).groups()[0] + return packaging.version.parse(default_version) + + +def pre_release_work(patch=False): + """Do all the necessary pre-release steps.""" + # First let's get the default version: base version if we are in dev, bump minor otherwise. + default_version = get_version() + if patch and default_version.is_devrelease: + raise ValueError("Can't create a patch version from the dev branch, checkout a released version!") + if default_version.is_devrelease: + default_version = default_version.base_version + elif patch: + default_version = f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}" + else: + default_version = f"{default_version.major}.{default_version.minor + 1}.0" + + # Now let's ask nicely if that's the right one. + version = input(f"Which version are you releasing? [{default_version}]") + if len(version) == 0: + version = default_version + + print(f"Updating version to {version}.") + global_version_update(version, patch=patch) + if not patch: + print("Cleaning main README") + clean_master_ref_in_model_list() + + +def update_custom_js(version, patch=False): + """Update the version table in the custom.js file.""" + with open(CUSTOM_JS_FILE, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + index = 0 + + # First let's put the right version + while not lines[index].startswith("const stableVersion ="): + index += 1 + lines[index] = f'const stableVersion = "v{version}"\n' + + # Then update the dictionary + while not lines[index].startswith("const versionMapping = {"): + index += 1 + + # We go until the end + while not lines[index].startswith("}"): + search = re.search(r'^(\s+)"": "([^"]+) \(stable\)",\s*\n$', lines[index]) + if search is not None: + indent, old_versions = search.groups() + if patch: + # We add the patch to the current stable doc + old_versions = f"{old_versions}/v{version}" + lines[index] = f'{indent}"": "{old_versions} (stable)",\n' + else: + # We only keep the last of the micro versions associated to that particular release + old_version = old_versions.split("/")[-1] + lines[index] = f'{indent}"": "v{version} (stable)",\n{indent}"{old_version}": "{old_versions}",\n' + index += 1 + + with open(CUSTOM_JS_FILE, "w", encoding="utf-8", newline="\n") as f: + lines = f.writelines(lines) + + +def update_deploy_sh(version, commit): + with open(DEPLOY_SH_FILE, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + + index = len(lines) - 1 + while len(lines[index]) <= 1: + index -= 1 + + search = re.search(r'^deploy_doc\s+"(\S+)"\s+#\s+(v\S+)\s+', lines[index]) + old_commit, old_version = search.groups() + lines[ + index + ] = f'deploy_doc "{old_commit}" {old_version}\ndeploy_doc "{commit}" # v{version} Latest stable release' + + with open(DEPLOY_SH_FILE, "w", encoding="utf-8", newline="\n") as f: + f.writelines(lines) + + +def post_release_work(): + """Do all the necesarry post-release steps.""" + # First let's get the current version + current_version = get_version() + dev_version = f"{current_version.major}.{current_version.minor + 1}.0.dev0" + current_version = current_version.base_version + # Get the current commit hash + repo = git.Repo(".", search_parent_directories=True) + version_commit = repo.head.object.hexsha[:7] + + # Check with the user we got that right. + version = input(f"Which version are we developing now? [{dev_version}]") + commit = input(f"Commit hash to associate to v{current_version}? [{version_commit}]") + if len(version) == 0: + version = dev_version + if len(commit) == 0: + commit = version_commit + + print(f"Updating version to {version}.") + global_version_update(version) + + print("Updating doc deployment and version navbar in the source documentation.") + update_custom_js(current_version) + update_deploy_sh(current_version, commit) + + +def post_patch_work(): + """Do all the necesarry post-patch steps.""" + # Try to guess the right info: last patch in the minor release before current version and its commit hash. + current_version = get_version() + repo = git.Repo(".", search_parent_directories=True) + repo_tags = repo.tags + default_version = None + version_commit = None + for tag in repo_tags: + if str(tag).startswith(f"v{current_version.major}.{current_version.minor - 1}"): + if default_version is None: + default_version = packaging.version.parse(str(tag)[1:]) + version_commit = str(tag.commit)[:7] + elif packaging.version.parse(str(tag)[1:]) > default_version: + default_version = packaging.version.parse(str(tag)[1:]) + version_commit = str(tag.commit)[:7] + + # Confirm with the user or ask for the info if not found. + if default_version is None: + version = input("Which patch version was just released?") + commit = input("Commit hash to associated to it?") + else: + version = input(f"Which patch version was just released? [{default_version}]") + commit = input(f"Commit hash to associated to it? [{version_commit}]") + if len(version) == 0: + version = default_version + if len(commit) == 0: + commit = version_commit + + print("Updating doc deployment and version navbar in the source documentation.") + update_custom_js(version, patch=True) + update_deploy_sh(version, commit) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.") + parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.") + args = parser.parse_args() + if not args.post_release: + pre_release_work(patch=args.patch) + elif args.patch: + post_patch_work() + else: + post_release_work()