name: Self-hosted runner (benchmark)

on:
  schedule:
    - cron: "17 2 * * *"
  workflow_call:

env:
  HF_HOME: /mnt/cache
  TF_FORCE_GPU_ALLOW_GROWTH: true


jobs:
  benchmark:
    name: Benchmark
    runs-on: 
      group: aws-g5-4xlarge-cache
    container:
      image: huggingface/transformers-all-latest-gpu
      options: --gpus all --privileged --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
    steps:
      - name: Update clone
        working-directory: /transformers
        run: |
          git fetch && git checkout ${{ github.sha }}

      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
        working-directory: /transformers
        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

      - name: Benchmark (daily)
        if: github.event_name == 'schedule'
        working-directory: /transformers
        run: |
          python3 -m pip install optimum-benchmark>=0.3.0
          HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun

      - name: Benchmark (merged to main event)
        if: github.event_name == 'push' && github.ref_name == 'main'
        working-directory: /transformers
        run: |
          python3 -m pip install optimum-benchmark>=0.3.0
          HF_TOKEN=${{ secrets.TRANSFORMERS_BENCHMARK_TOKEN }} python3 benchmark/benchmark.py --repo_id hf-internal-testing/benchmark_results_merge_event --path_in_repo $(date +'%Y-%m-%d') --config-dir benchmark/config --config-name generation --commit=${{ github.sha }} backend.model=google/gemma-2b backend.cache_implementation=null,static backend.torch_compile=false,true --multirun