Enable benchmarking

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "monthly"

.github/workflows/asv-main.yml

@@ -0,0 +1,68 @@
+# This workflow will run benchmarks with airspeed velocity (asv), 
+# store the new results in the "benchmarks" branch and publish them
+# to a dashboard on GH Pages.
+name: Run ASV benchmarks for main
+
+on:
+  push:
+    branches: [ main ]
+
+env:
+  PYTHON_VERSION: "3.11"
+  ASV_VERSION: "0.6.5"
+  WORKING_DIR: ${{github.workspace}}/benchmarks
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  asv-main:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    defaults:
+      run:
+        working-directory: ${{env.WORKING_DIR}}
+    steps:
+    - name: Set up Python ${{env.PYTHON_VERSION}}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{env.PYTHON_VERSION}}
+    - name: Checkout main branch of the repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Install dependencies
+      run: pip install asv[virtualenv]==${{env.ASV_VERSION}}
+    - name: Configure git
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+    - name: Create ASV machine config file
+      run: asv machine --machine gh-runner --yes
+    - name: Fetch previous results from the "benchmarks" branch
+      run: |
+        if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
+          git merge origin/benchmarks \
+            --allow-unrelated-histories \
+            --no-commit
+          mv ../_results .
+        fi
+    - name: Run ASV for the main branch
+      run: asv run ALL --skip-existing --verbose || true
+    - name: Submit new results to the "benchmarks" branch
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        branch: benchmarks
+        folder: ${{env.WORKING_DIR}}/_results
+        target-folder: _results
+    - name: Generate dashboard HTML
+      run: |
+        asv show
+        asv publish
+    - name: Deploy to Github pages
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        branch: gh-pages
+        folder: ${{env.WORKING_DIR}}/_html

.github/workflows/asv-nightly.yml

@@ -0,0 +1,72 @@
+# This workflow will run daily at 06:45.
+# It will run benchmarks with airspeed velocity (asv)
+# and compare performance with the previous nightly build.
+name: Run benchmarks nightly job
+
+on:
+  schedule:
+    - cron: 45 6 * * *
+  workflow_dispatch:
+
+env:
+  PYTHON_VERSION: "3.11"
+  ASV_VERSION: "0.6.5"
+  WORKING_DIR: ${{github.workspace}}/benchmarks
+  NIGHTLY_HASH_FILE: nightly-hash
+
+jobs:
+  asv-nightly:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{env.WORKING_DIR}}
+    steps:
+    - name: Set up Python ${{env.PYTHON_VERSION}}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{env.PYTHON_VERSION}}
+    - name: Checkout main branch of the repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Install dependencies
+      run: pip install asv[virtualenv]==${{env.ASV_VERSION}}
+    - name: Configure git
+      run: |
+        git config user.name "github-actions[bot]"
+        git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+    - name: Create ASV machine config file
+      run: asv machine --machine gh-runner --yes
+    - name: Fetch previous results from the "benchmarks" branch
+      run: |
+        if git ls-remote --exit-code origin benchmarks > /dev/null 2>&1; then
+          git merge origin/benchmarks \
+            --allow-unrelated-histories \
+            --no-commit
+          mv ../_results .
+        fi
+    - name: Get nightly dates under comparison
+      id: nightly-dates
+      run: |
+        echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+        echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+    - name: Use last nightly commit hash from cache
+      uses: actions/cache@v4
+      with:
+        path: ${{env.WORKING_DIR}}
+        key: nightly-results-${{steps.nightly-dates.outputs.yesterday}}
+    - name: Run comparison of main against last nightly build
+      run: |
+        HASH_FILE=${{env.NIGHTLY_HASH_FILE}}
+        CURRENT_HASH=${{github.sha}}
+        if [ -f $HASH_FILE ]; then
+          PREV_HASH=$(cat $HASH_FILE)
+          asv continuous $PREV_HASH $CURRENT_HASH --verbose || true
+          asv compare $PREV_HASH $CURRENT_HASH --sort ratio --verbose
+        fi
+        echo $CURRENT_HASH > $HASH_FILE
+    - name: Update last nightly hash in cache
+      uses: actions/cache@v4
+      with:
+        path: ${{env.WORKING_DIR}}
+        key: nightly-results-${{steps.nightly-dates.outputs.today}}

.github/workflows/asv-pr.yml

@@ -0,0 +1,70 @@
+# This workflow will run benchmarks with airspeed velocity (asv) for pull requests.
+# It will compare the performance of the main branch with the performance of the merge
+# with the new changes. It then publishes a comment with this assessment by triggering
+# the publish-benchmarks-pr workflow.
+# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
+name: Run benchmarks for PR
+
+on:
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: "3.11"
+  ASV_VERSION: "0.6.5"
+  WORKING_DIR: ${{github.workspace}}/benchmarks
+  ARTIFACTS_DIR: ${{github.workspace}}/artifacts
+
+jobs:
+  asv-pr:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{env.WORKING_DIR}}
+    steps:
+    - name: Set up Python ${{env.PYTHON_VERSION}}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{env.PYTHON_VERSION}}
+    - name: Checkout PR branch of the repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Display Workflow Run Information
+      run: |
+        echo "Workflow Run ID: ${{github.run_id}}"
+    - name: Install dependencies
+      run: pip install asv[virtualenv]==${{env.ASV_VERSION}} lf-asv-formatter
+    - name: Make artifacts directory
+      run: mkdir -p ${{env.ARTIFACTS_DIR}}
+    - name: Save pull request number
+      run: echo ${{github.event.pull_request.number}} > ${{env.ARTIFACTS_DIR}}/pr
+    - name: Get current job logs URL
+      uses: Tiryoh/gha-jobid-action@v1
+      id: jobs
+      with:
+        github_token: ${{secrets.GITHUB_TOKEN}}
+        job_name: ${{github.job}}
+    - name: Create ASV machine config file
+      run: asv machine --machine gh-runner --yes
+    - name: Save comparison of PR against main branch
+      run: |
+        git remote add upstream https://github.com/${{github.repository}}.git
+        git fetch upstream
+        asv continuous upstream/main HEAD --verbose || true
+        asv compare upstream/main HEAD --sort ratio --verbose | tee output
+        python -m lf_asv_formatter --asv_version "$(asv --version | awk '{print $2}')"
+        printf "\n\nClick [here]($STEP_URL) to view all benchmarks." >> output
+        mv output ${{env.ARTIFACTS_DIR}}
+      env:
+        STEP_URL: ${{steps.jobs.outputs.html_url}}#step:10:1
+    - name: Upload artifacts (PR number and benchmarks output)
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark-artifacts
+        path: ${{env.ARTIFACTS_DIR}}

.github/workflows/publish-benchmarks-pr.yml

@@ -0,0 +1,53 @@
+# This workflow publishes a benchmarks comment on a pull request. It is triggered after the
+# benchmarks are computed in the asv-pr workflow. This separation of concerns allows us limit
+# access to the target repository private tokens and secrets, increasing the level of security.
+# Based on https://securitylab.github.com/research/github-actions-preventing-pwn-requests/.
+name: Publish benchmarks comment to PR
+
+on:
+  workflow_run:
+    workflows: ["Run benchmarks for PR"]
+    types: [completed]
+
+jobs:
+  upload-pr-comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - name: Display Workflow Run Information
+      run: |
+        echo "Workflow Run ID: ${{ github.event.workflow_run.id }}"
+        echo "Head SHA: ${{ github.event.workflow_run.head_sha }}"
+        echo "Head Branch: ${{ github.event.workflow_run.head_branch }}"
+        echo "Conclusion: ${{ github.event.workflow_run.conclusion }}"
+        echo "Event: ${{ github.event.workflow_run.event }}"
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v7
+      with:
+        name: benchmark-artifacts
+        run_id: ${{ github.event.workflow_run.id }}
+    - name: Extract artifacts information
+      id: pr-info
+      run: |
+        printf "PR number: $(cat pr)\n"
+        printf "Output:\n$(cat output)"
+        printf "pr=$(cat pr)" >> $GITHUB_OUTPUT
+    - name: Find benchmarks comment
+      uses: peter-evans/find-comment@v3
+      id: find-comment
+      with:
+        issue-number: ${{ steps.pr-info.outputs.pr }}
+        comment-author: 'github-actions[bot]'
+        body-includes: view all benchmarks
+    - name: Create or update benchmarks comment
+      uses: peter-evans/create-or-update-comment@v4
+      with:
+        comment-id: ${{ steps.find-comment.outputs.comment-id }}
+        issue-number: ${{ steps.pr-info.outputs.pr }}
+        body-path: output
+        edit-mode: replace

.gitignore

@@ -26,7 +26,11 @@ docs/api/*
 .spyproject/*
 .ipynb_checkpoints
 
+# Ignore the results and html folders generated by benchmarks
+/benchmarks/_results
+/benchmarks/_html
+
 # Coverage files
 .coverage
 coverage.xml
-coverage_output*.txt
+coverage_output*.txt

benchmarks/README.md

@@ -0,0 +1,12 @@
+# Benchmarks
+
+This directory contains files that will be run via continuous testing either
+nightly or after committing code to a pull request. 
+
+The runtime and/or memory usage of the functions defined in these files will be
+tracked and reported to give you a sense of the overall performance of your code.
+
+You are encouraged to add, update, or remove benchmark functions to suit the needs
+of your project.
+
+For more information, see the documentation here: https://lincc-ppt.readthedocs.io/en/latest/practices/ci_benchmarking.html

benchmarks/asv.conf.json

@@ -0,0 +1,81 @@
+
+{
+    // The version of the config file format. Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+    // The name of the project being benchmarked.
+    "project": "redback",
+    // The project's homepage.
+    "project_url": "https://github.com/nikhil-sarin/redback",
+    // The URL or local path of the source code repository for the
+    // project being benchmarked.
+    "repo": "..",
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "tip" (for mercurial).
+    "branches": [
+        "HEAD"
+    ],
+    "install_command": [
+        "python -m pip install {wheel_file}"
+    ],
+    "build_command": [
+        "python -m build --wheel -o {build_cache_dir} {build_dir}"
+    ],
+    // The DVCS being used. If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    "dvcs": "git",
+    // The tool to use to create environments. May be "conda",
+    // "virtualenv" or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "virtualenv",
+    // the base URL to show a commit for the project.
+    "show_commit_url": "https://github.com/nikhil-sarin/redback/commit/",
+    // The Pythons you'd like to test against. If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    "pythons": [
+        "3.11"
+    ],
+    // The matrix of dependencies to test. Each key is the name of a
+    // package (in PyPI) and the values are version numbers.  An empty
+    // list indicates to just test against the default (latest)
+    // version.
+    "matrix": {
+        "Cython": [],
+        "build": [],
+        "packaging": []
+    },
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in. If not provided, defaults to "benchmarks".
+    "benchmark_dir": ".",
+    // The directory (relative to the current directory) to cache the Python
+    // environments in. If not provided, defaults to "env".
+    "env_dir": "env",
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in. If not provided, defaults to "results".
+    "results_dir": "_results",
+    // The directory (relative to the current directory) that the html tree
+    // should be written to. If not provided, defaults to "html".
+    "html_dir": "_html",
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+    // `asv` will cache wheels of the recent builds in each
+    // environment, making them faster to install next time. This is
+    // number of builds to keep, per environment.
+    "build_cache_size": 8
+    // The commits after which the regression search in `asv publish`
+    // should start looking for regressions. Dictionary whose keys are
+    // regexps matching to benchmark names, and values corresponding to
+    // the commit (exclusive) after which to start looking for
+    // regressions. The default is to start from the first commit
+    // with results. If the commit is `null`, regression detection is
+    // skipped for the matching benchmark.
+    //
+    // "regressions_first_commits": {
+    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
+    //    "another_benchmark": null,   // Skip regression detection altogether
+    // }
+}