riaz.somc/transformers
1
0
Fork 0
mirror of https://github.com/huggingface/transformers.git synced 2025-07-04 05:10:06 +06:00
Code Issues Actions 20 Packages Projects Releases Wiki Activity

No self-hosted runner for dev documentation (#15710)

Browse Source
This commit is contained in:
Lysandre Debut 2022-03-01 20:05:54 +01:00 committed by GitHub
parent 00eaffc81f
commit 26426923b7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 82 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
.github/workflows/build-docker-images.yml vendored
View File

@ -66,6 +66,30 @@ jobs:
push: true
tags: huggingface/transformers-pytorch-deepspeed-latest-gpu
doc-builder:
name: "Doc builder"
runs-on: ubuntu-latest
steps:
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
-
name: Check out code
uses: actions/checkout@v2
-
name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
name: Build and push
uses: docker/build-push-action@v2
with:
context: ./docker/transformers-doc-builder
push: true
tags: huggingface/transformers-doc-builder
latest-pytorch:
name: "Latest PyTorch [dev]"
runs-on: ubuntu-latest

98
.github/workflows/build_dev_documentation.yml vendored
View File

@ -3,13 +3,15 @@ name: Build dev documentation
on:
pull_request:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
build_and_package:
runs-on: [self-hosted, doc-builder]
runs-on: ubuntu-latest
container:
image: huggingface/doc-builder-transformers
options: "-v /home/github_actions:/mnt"
image: huggingface/transformers-doc-builder
env:
PR_NUMBER: ${{ github.event.number }}
EVENT_CONTEXT: ${{ toJSON(github.event) }}
@ -35,7 +37,33 @@ jobs:
node-version: '16'
- name: Set env
run: echo "WRITE=$(cat /mnt/WRITE)" >> $GITHUB_ENV
run: |
echo "WRITE=$(echo 'ghp_'$(wget -qO- lysand.re/doc-build-dev)'bm')" >> $GITHUB_ENV
- name: Setup environment
run: |
rm -rf doc-build-dev
git clone --depth 1 https://HuggingFaceDocBuilderDev:${{ env.WRITE }}@github.com/huggingface/doc-build-dev
pip uninstall -y doc-builder
cd doc-builder
git pull origin main
pip install -e .
cd ..
cd transformers
pip install .[dev]
cd ..
cd notebooks
git pull origin master
cd ..
- name: Setup git
run: |
git config --global user.name "Hugging Face Doc Builder"
git config --global user.email docs@huggingface.co
- name: Comment PR
uses: thollander/actions-comment-pull-request@v1
@ -63,61 +91,19 @@ jobs:
body: |
The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/transformers/pr_${{ env.PR_NUMBER }}). All of your documentation changes will be reflected on that endpoint.
- name: Loading cache.
uses: actions/cache@v2
id: cache
with:
path: ~/.cache/pip
key: v1-test_build_doc
restore-keys: |
v1-test_build_doc-${{ hashFiles('setup.py') }}
v1-test_build_doc
- name: Setup environment
run: |
apt-get -y update && apt-get install -y libsndfile1-dev
pip uninstall -y doc-builder
pip install git+https://github.com/huggingface/doc-builder -U
cd transformers
pip install .[dev]
cd ..
export TORCH_VERSION=$(python -c "from torch import version; print(version.__version__.split('+')[0])")
pip install torch-scatter -f https://data.pyg.org/whl/torch-${TORCH_VERSION}+cpu.html
pip install torchvision
python -m pip install 'git+https://github.com/facebookresearch/detectron2.git'
apt install -y tesseract-ocr
pip install pytesseract
pip install pytorch-quantization --extra-index-url https://pypi.ngc.nvidia.com
pip install https://github.com/kpu/kenlm/archive/master.zip
- name: Setup git
run: |
git config --global user.name "Hugging Face Doc Builder"
git config --global user.email docs@huggingface.co
cd notebooks
git pull origin master
cd ..
WRITE=`cat /mnt/WRITE`
rm -rf doc-build-dev
git clone https://HuggingFaceDocBuilder:$WRITE@github.com/huggingface/doc-build-dev
- name: Make documentation
env:
NODE_OPTIONS: --max-old-space-size=6656
run: |
cd doc-builder &&
doc-builder build transformers ../transformers/docs/source --build_dir ../doc-build-dev --notebook_dir notebooks/transformers_doc --clean --version pr_$PR_NUMBER --html &&
cd ..
cd doc-build-dev && git pull
cd ../doc-builder
doc-builder build transformers ../transformers/docs/source --build_dir ../doc-build-dev --notebook_dir ../notebooks/transformers_doc --clean --version pr_$PR_NUMBER --html
cd ../doc-build-dev && git stash && git pull && git stash apply && cd ..
- name: Push to repositories
run: |
cd doc-build-dev &&
ls &&
git add . &&
git commit -m "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/transformers/commit/${{ github.sha }}" &&
cd doc-build-dev
ls
git add .
git commit -m "Updated with commit ${{ github.sha }} See: https://github.com/huggingface/transformers/commit/${{ github.sha }}"
git push origin main

16
docker/transformers-doc-builder/Dockerfile Normal file
View File

@ -0,0 +1,16 @@
FROM python:3.8
LABEL maintainer="Hugging Face"
RUN apt update
RUN git clone https://github.com/huggingface/transformers
RUN python3 -m pip install --no-cache-dir --upgrade pip && python3 -m pip install --no-cache-dir git+https://github.com/huggingface/doc-builder ./transformers[dev,deepspeed]
RUN apt-get -y update && apt-get install -y libsndfile1-dev && apt install -y tesseract-ocr
RUN python3 -m pip install --no-cache-dir torch-scatter -f https://data.pyg.org/whl/torch-$(python -c "from torch import version; print(version.__version__.split('+')[0])")+cpu.html
RUN python3 -m pip install --no-cache-dir torchvision git+https://github.com/facebookresearch/detectron2.git pytesseract https://github.com/kpu/kenlm/archive/master.zip
RUN python3 -m pip install --no-cache-dir pytorch-quantization --extra-index-url https://pypi.ngc.nvidia.com
RUN python3 -m pip install -U "itsdangerous<2.1.0"
RUN doc-builder build transformers transformers/docs/source --build_dir doc-build-dev --notebook_dir notebooks/transformers_doc --clean --version pr_$PR_NUMBER
RUN rm -rf doc-build-dev