eval-overhead-e2e.yml

name: Instrumentor Benchmark
on: 
  push:
    branches:
      - main
    paths:
      - '.github/workflows/**'
      - 'traincheck/instrumentor/**'
      - 'traincheck/collect_trace.py'
  pull_request:
    paths:
      - '.github/workflows/**'
      - 'traincheck/instrumentor/**'
      - 'traincheck/collect_trace.py'
    

permissions:
  contents: write
  deployments: write
  pull-requests: write

jobs:
  benchmark:
    runs-on: self-hosted

    name: Run Overhead Benchmarks
    steps:
      - uses: actions/checkout@v4
        name: Checkout TrainCheck
      
      - uses: actions/checkout@v4
        name: Checkout TrainCheck-Benchmarks
        with:
          repository: OrderLab/TrainCheck-Benchmarks
          path: benchmarks
      
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
    
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r benchmarks/requirements.txt
          pip install -r benchmarks/requirements-torch222-cuda121.txt
          pip install -e .

      - name: Run benchmark
        run: |
          cd benchmarks/instr-overhead
          xonsh run_all.xsh --track-variables --res_folder perf_benchmark_results
      
      - name: Run analysis scripts
        run: |
          cd benchmarks/instr-overhead
          xonsh analysis.xsh --res_folder perf_benchmark_results
          cat perf_benchmark_results/overhead_e2e.csv

      - name: Generate plots
        run: |
          cd benchmarks/instr-overhead
          COMMIT_HASH=$(git rev-parse --short HEAD)
          COMMIT_MESSAGE=$(git log -1 --pretty=%B)
          python3 plot_e2e.py -i perf_benchmark_results/overhead_e2e.csv -o ./overhead_macro.png -t "$COMMIT_HASH - $COMMIT_MESSAGE"

      - name: Update docs with benchmark plots
        run: |
          TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M:%S UTC")
          COMMIT_HASH=$(git rev-parse --short HEAD)
          
          # Copy plots to docs assets
          mkdir -p docs/assets/images
          cp benchmarks/instr-overhead/overhead_macro.png docs/assets/images/
          
          # Copy CSV to docs assets
          mkdir -p docs/assets/csv
          cp benchmarks/instr-overhead/perf_benchmark_results/overhead_e2e.csv docs/assets/csv/
          
          # Create or update benchmark page
          cat > docs/benchmarks.md << EOF
          # Performance Benchmarks
          
          Latest benchmark results (updated: $TIMESTAMP, commit: $COMMIT_HASH)
          
          ## End-to-End Performance Impact
          ![E2E Benchmark Results](assets/images/overhead_macro.png)
          
          These benchmarks are automatically generated from the [TrainCheck-Benchmarks](https://github.com/OrderLab/TrainCheck-Benchmarks) repository.
          EOF
          
          # Commit and push changes
          git config user.email "jyuxuan@umich.edu"
          git config user.name "Yuxuan Jiang"
          git add docs/assets/images/overhead_macro.png
          git add docs/assets/csv/overhead_e2e.csv
          git add docs/benchmarks.md
          git commit -m "Update benchmark plots for $COMMIT_HASH" || echo "No changes to commit"
          git push origin main

      - name: Deploy docs
        run: |
          pip install mkdocs-material
          mkdocs gh-deploy --force