Skip to content

Add Windows build for PyPI wheels #2612

Add Windows build for PyPI wheels

Add Windows build for PyPI wheels #2612

# (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
#
name: PyPI Wheels
on:
# Build wheels on every push to test the build process
push:
branches:
- "**"
paths-ignore:
- "**/website/**"
tags:
- 'v*'
# Build on pull requests
pull_request:
branches:
- "**"
paths-ignore:
- "**/website/**"
# Allow manual trigger for testing builds or publishing
workflow_dispatch:
inputs:
publish:
description: 'Publish to PyPI (otherwise just build and test)'
required: false
type: boolean
default: false
test_pypi:
description: 'Publish to TestPyPI instead of PyPI (for testing)'
required: false
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
# Linux CPU wheels are built with cibuildwheel (for manylinux_2_28 compatibility)
build_cpu_wheels_linux:
name: pypi-cpu-linux (cibuildwheel)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
submodules: recursive
fetch-depth: 0 # Fetch full history for setuptools_scm version detection
fetch-tags: true # Ensure all tags are fetched
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install Jinja2
run: pip install jinja2
- name: Generate pyproject.toml
run: |
python scripts/generate_pyproject.py
# Set a default pyproject.toml for the initial setup
cp pyproject-pypi-cpu-py312.toml pyproject.toml
- name: Build wheels
uses: pypa/cibuildwheel@v3.3.1
- uses: actions/upload-artifact@v6
with:
name: wheels-cpu-linux
path: ./wheelhouse/*.whl
# CPU wheels are built with pixi (non-Linux platforms)
# Linux CPU wheels are handled by build_cpu_wheels_linux above using cibuildwheel
build_cpu_wheels:
name: pypi-cpu-py${{ matrix.python-version }}-${{ matrix.os-short }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
matrix:
os: [macos-14, windows-latest]
python-version: ['3.12', '3.13']
include:
- python-version: '3.12'
pixi-environment: py312
- python-version: '3.13'
pixi-environment: py313
- os: macos-14
os-short: mac-arm64
- os: windows-latest
os-short: win64
steps:
# Skip Python 3.13 builds on regular commits to reduce CI cost
# Python 3.13 builds will still run when publishing (tags or manual workflow_dispatch with publish=true)
- name: Check if should run
id: should_run
run: |
if [[ "${{ matrix.python-version }}" == "3.13" ]]; then
if [[ "${{ startsWith(github.ref, 'refs/tags/v') }}" == "true" ]] || \
[[ "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' }}" == "true" ]]; then
echo "run=true" >> $GITHUB_OUTPUT
else
echo "run=false" >> $GITHUB_OUTPUT
fi
else
echo "run=true" >> $GITHUB_OUTPUT
fi
shell: bash
- uses: actions/checkout@v6
if: steps.should_run.outputs.run == 'true'
with:
submodules: recursive
fetch-depth: 0 # Fetch full history for setuptools_scm version detection
fetch-tags: true # Ensure all tags are fetched
# ccache for macOS
- name: Set up ccache (macOS)
if: steps.should_run.outputs.run == 'true' && runner.os == 'macOS'
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ${{ github.workflow }}-${{ matrix.os }}-py${{ matrix.python-version }}
# sccache for Windows
- name: Set up sccache (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
uses: mozilla-actions/sccache-action@v0.0.9
- name: Set up pixi
if: steps.should_run.outputs.run == 'true'
uses: prefix-dev/setup-pixi@v0.9.3
with:
pixi-version: latest
cache: true
cache-write: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
environments: ${{ matrix.pixi-environment }}
- name: Generate pyproject configs
if: steps.should_run.outputs.run == 'true'
run: pixi run -e ${{ matrix.pixi-environment }} generate_pyproject
- name: Clean distribution artifacts
if: steps.should_run.outputs.run == 'true'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_clean
- name: Build CPU wheel
if: steps.should_run.outputs.run == 'true'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_build
env:
# Enable compiler caching: ccache on macOS, sccache on Windows
CMAKE_C_COMPILER_LAUNCHER: ${{ runner.os == 'Windows' && 'sccache' || 'ccache' }}
CMAKE_CXX_COMPILER_LAUNCHER: ${{ runner.os == 'Windows' && 'sccache' || 'ccache' }}
SCCACHE_GHA_ENABLED: ${{ runner.os == 'Windows' && 'true' || '' }}
# Repair wheel on Windows using delvewheel (bundles DLLs)
- name: Repair wheel (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_repair
- name: Print ccache stats (macOS)
if: steps.should_run.outputs.run == 'true' && runner.os == 'macOS'
run: ccache -s
- name: Print sccache stats (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
run: sccache --show-stats
- name: Upload wheel artifacts
if: steps.should_run.outputs.run == 'true'
uses: actions/upload-artifact@v6
with:
name: wheels-cpu-${{ matrix.os }}-py${{ matrix.python-version }}
path: dist/*.whl
retention-days: 7
build_gpu_wheels:
name: pypi-gpu-py${{ matrix.python-version }}-${{ matrix.os-short }}
runs-on: ${{ matrix.os }}
strategy:
fail-fast: true
# Run GPU builds sequentially to avoid disk space exhaustion
# Each PyTorch+CUDA install needs ~3GB and container overlay shares host disk
max-parallel: 1
matrix:
os: [ubuntu-latest, windows-latest]
python-version: ['3.12', '3.13']
include:
- python-version: '3.12'
cuda-version: "12.9.0"
- python-version: '3.13'
cuda-version: "12.9.0"
- os: ubuntu-latest
os-short: ubuntu
- os: windows-latest
os-short: win64
# Linux uses cibuildwheel-based GPU environments
- os: ubuntu-latest
python-version: '3.12'
pixi-environment: gpu-wheel-build-py312
- os: ubuntu-latest
python-version: '3.13'
pixi-environment: gpu-wheel-build-py313
# Windows uses pip wheel + delvewheel with CUDA from conda-forge
- os: windows-latest
python-version: '3.12'
pixi-environment: gpu-wheel-build-win-py312
- os: windows-latest
python-version: '3.13'
pixi-environment: gpu-wheel-build-win-py313
env:
FULL_CUDA_VERSION: ${{ matrix.cuda-version }}
# Mock CUDA virtual package so pixi can install GPU environments on CI runners without physical GPUs
# This allows conda-forge CUDA toolkit packages to be installed for cross-compilation
CONDA_OVERRIDE_CUDA: "12.0"
steps:
# Skip Python 3.13 builds on regular commits to reduce CI cost
# Python 3.13 builds will still run when publishing (tags or manual workflow_dispatch with publish=true)
- name: Check if should run
id: should_run
run: |
if [[ "${{ matrix.python-version }}" == "3.13" ]]; then
if [[ "${{ startsWith(github.ref, 'refs/tags/v') }}" == "true" ]] || \
[[ "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' }}" == "true" ]]; then
echo "run=true" >> $GITHUB_OUTPUT
else
echo "run=false" >> $GITHUB_OUTPUT
fi
else
echo "run=true" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Free disk space
if: steps.should_run.outputs.run == 'true' && matrix.os == 'ubuntu-latest'
uses: jlumbroso/free-disk-space@main
with:
# Remove all default tools and applications
tool-cache: false # Keep tool cache for Python
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: false # Keep container images for cibuildwheel
swap-storage: true
- name: Maximize build space (Ubuntu only)
# DISABLED: This action restructures the filesystem and breaks container engine functionality
# The "Free disk space" step above provides enough space for the build
if: false && steps.should_run.outputs.run == 'true' && matrix.os == 'ubuntu-latest'
uses: easimon/maximize-build-space@master
with:
root-reserve-mb: 30720
swap-size-mb: 1024
remove-dotnet: true
remove-android: true
remove-haskell: true
remove-codeql: true
remove-docker-images: false # Keep container images for cibuildwheel
- uses: actions/checkout@v6
if: steps.should_run.outputs.run == 'true'
with:
submodules: recursive
fetch-depth: 0 # Fetch full history for setuptools_scm version detection
fetch-tags: true # Ensure all tags are fetched
- name: Set up ccache (Linux)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Linux'
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ${{ github.workflow }}-${{ matrix.os }}-py${{ matrix.python-version }}-gpu
- name: Set up sccache (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
uses: mozilla-actions/sccache-action@v0.0.9
# Note: No separate CUDA toolkit install needed for Windows.
# conda-forge's cuda-toolkit (via pixi) provides nvcc and CUDA headers/libs.
# We use Ninja generator (set in pyproject-pypi.toml.j2) to avoid MSBuild CUDA
# .targets PATH issues. CMAKE_CUDA_HOST_COMPILER points nvcc to cl.exe directly.
- name: Find MSVC cl.exe path (Windows GPU)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
id: msvc
shell: bash
run: |
# Find vswhere.exe (installed by default on GitHub-hosted Windows runners)
VSWHERE="/c/Program Files (x86)/Microsoft Visual Studio/Installer/vswhere.exe"
if [ ! -f "$VSWHERE" ]; then
echo "::error::vswhere.exe not found"
exit 1
fi
# Get the Visual Studio installation path
VS_PATH=$("$VSWHERE" -latest -property installationPath)
echo "Visual Studio path: $VS_PATH"
# Find cl.exe for the host architecture (x64)
CL_PATH=$(find "$VS_PATH/VC/Tools/MSVC" -name "cl.exe" -path "*/Hostx64/x64/*" | head -1)
if [ -z "$CL_PATH" ]; then
echo "::error::cl.exe not found in Visual Studio installation"
exit 1
fi
echo "cl.exe found: $CL_PATH"
# Convert to Windows path format for CMake
CL_PATH_WIN=$(cygpath -w "$CL_PATH")
echo "cl.exe (Windows path): $CL_PATH_WIN"
echo "cl_path=$CL_PATH_WIN" >> $GITHUB_OUTPUT
# Also export the directory containing cl.exe for PATH
CL_DIR=$(dirname "$CL_PATH")
CL_DIR_WIN=$(cygpath -w "$CL_DIR")
echo "cl_dir=$CL_DIR_WIN" >> $GITHUB_OUTPUT
- name: Set up pixi
if: steps.should_run.outputs.run == 'true'
uses: prefix-dev/setup-pixi@v0.9.3
with:
pixi-version: latest
cache: true
cache-write: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
environments: ${{ matrix.pixi-environment }}
- name: Generate pyproject configs
if: steps.should_run.outputs.run == 'true'
run: pixi run -e ${{ matrix.pixi-environment }} generate_pyproject
- name: Clean distribution artifacts
if: steps.should_run.outputs.run == 'true'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_clean
- name: Determine version for container builds (Linux)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Linux'
id: get_version
run: |
# Install setuptools_scm to determine version from git tags
pip install setuptools_scm
# Get the version that setuptools_scm would generate
FULL_VERSION=$(python -c "from setuptools_scm import get_version; print(get_version())")
echo "Detected version: $FULL_VERSION"
# Strip local version part (e.g., +gad8997da8) - PyPI doesn't allow local versions
# This converts "0.1.101.dev26+gad8997da8" to "0.1.101.dev26"
VERSION=$(echo "$FULL_VERSION" | sed 's/+.*//')
echo "Version for PyPI: $VERSION"
echo "version=$VERSION" >> $GITHUB_OUTPUT
- name: Build GPU wheel (Linux)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Linux'
timeout-minutes: 60
env:
CIBW_ENVIRONMENT_PASS_LINUX: SETUPTOOLS_SCM_PRETEND_VERSION
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ steps.get_version.outputs.version }}
CMAKE_C_COMPILER_LAUNCHER: ccache
CMAKE_CXX_COMPILER_LAUNCHER: ccache
run: |
echo "=== Starting GPU wheel build ==="
echo "Environment: ${{ matrix.pixi-environment }}"
echo "Version: $SETUPTOOLS_SCM_PRETEND_VERSION"
echo "=== Detecting container engine ==="
source scripts/detect_container_engine.sh
$CIBW_CONTAINER_ENGINE --version || echo "Container engine not available"
$CIBW_CONTAINER_ENGINE info || echo "Container engine info failed"
echo "=== Running wheel_build ==="
pixi run -e ${{ matrix.pixi-environment }} wheel_build
echo "=== Build completed ==="
- name: Build GPU wheel (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
timeout-minutes: 90
env:
CMAKE_C_COMPILER_LAUNCHER: sccache
CMAKE_CXX_COMPILER_LAUNCHER: sccache
SCCACHE_GHA_ENABLED: "true"
CMAKE_CUDA_HOST_COMPILER: ${{ steps.msvc.outputs.cl_path }}
run: pixi run -e ${{ matrix.pixi-environment }} wheel_build
- name: Repair GPU wheel (Linux)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Linux'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_repair
- name: Repair GPU wheel (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
run: pixi run -e ${{ matrix.pixi-environment }} wheel_repair
- name: Print ccache stats (Linux)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Linux'
run: ccache -s
- name: Upload wheel artifacts
if: steps.should_run.outputs.run == 'true'
uses: actions/upload-artifact@v6
with:
name: wheels-gpu-${{ matrix.os }}-py${{ matrix.python-version }}
path: dist/*.whl
retention-days: 7
# Regression test: Verify pip wheels work correctly (import test, parallel operations)
# Tests all build variants with same skip logic as builds (py3.13 only on tags/releases)
test_pip_wheels:
name: Test pip wheel - ${{ matrix.variant }}-py${{ matrix.python-version }}-${{ matrix.os-short }}
needs: [build_cpu_wheels_linux, build_cpu_wheels, build_gpu_wheels]
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
# Linux CPU - always test py3.12
- os: ubuntu-latest
os-short: linux
variant: cpu
python-version: '3.12'
pixi-environment: py312
artifact-pattern: wheels-cpu-linux
wheel-type: cpu
# Linux CPU - py3.13 only on tags/releases
- os: ubuntu-latest
os-short: linux
variant: cpu
python-version: '3.13'
pixi-environment: py313
artifact-pattern: wheels-cpu-linux
wheel-type: cpu
# macOS - always test py3.12
- os: macos-14
os-short: mac-arm64
variant: cpu
python-version: '3.12'
pixi-environment: py312
artifact-pattern: wheels-cpu-macos-14-py3.12
wheel-type: cpu
# macOS - py3.13 only on tags/releases
- os: macos-14
os-short: mac-arm64
variant: cpu
python-version: '3.13'
pixi-environment: py313
artifact-pattern: wheels-cpu-macos-14-py3.13
wheel-type: cpu
# Windows - always test py3.12
- os: windows-latest
os-short: win64
variant: cpu
python-version: '3.12'
pixi-environment: py312
artifact-pattern: wheels-cpu-windows-latest-py3.12
wheel-type: cpu
# Windows - py3.13 only on tags/releases
- os: windows-latest
os-short: win64
variant: cpu
python-version: '3.13'
pixi-environment: py313
artifact-pattern: wheels-cpu-windows-latest-py3.13
wheel-type: cpu
# Linux GPU - always test py3.12 (import-only, no GPU required)
- os: ubuntu-latest
os-short: linux
variant: gpu
python-version: '3.12'
pixi-environment: py312
artifact-pattern: wheels-gpu-ubuntu-latest-py3.12
wheel-type: gpu
# Linux GPU - py3.13 only on tags/releases
- os: ubuntu-latest
os-short: linux
variant: gpu
python-version: '3.13'
pixi-environment: py313
artifact-pattern: wheels-gpu-ubuntu-latest-py3.13
wheel-type: gpu
# Windows GPU - always test py3.12 (import-only, no GPU required)
- os: windows-latest
os-short: win64
variant: gpu
python-version: '3.12'
pixi-environment: py312
artifact-pattern: wheels-gpu-windows-latest-py3.12
wheel-type: gpu
# Windows GPU - py3.13 only on tags/releases
- os: windows-latest
os-short: win64
variant: gpu
python-version: '3.13'
pixi-environment: py313
artifact-pattern: wheels-gpu-windows-latest-py3.13
wheel-type: gpu
steps:
# Skip py3.13 tests on regular commits (same logic as builds)
- name: Check if should run
id: should_run
run: |
if [[ "${{ matrix.python-version }}" == "3.13" ]]; then
if [[ "${{ startsWith(github.ref, 'refs/tags/v') }}" == "true" ]] || \
[[ "${{ github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' }}" == "true" ]]; then
echo "run=true" >> $GITHUB_OUTPUT
else
echo "run=false" >> $GITHUB_OUTPUT
fi
else
echo "run=true" >> $GITHUB_OUTPUT
fi
shell: bash
- uses: actions/checkout@v6
if: steps.should_run.outputs.run == 'true'
with:
submodules: recursive
- name: Set up pixi
if: steps.should_run.outputs.run == 'true'
uses: prefix-dev/setup-pixi@v0.9.3
with:
pixi-version: latest
cache: true
cache-write: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
environments: ${{ matrix.pixi-environment }}
- name: Download wheel artifacts
if: steps.should_run.outputs.run == 'true'
uses: actions/download-artifact@v7
with:
path: dist
pattern: ${{ matrix.artifact-pattern }}
merge-multiple: true
- name: List downloaded wheels
if: steps.should_run.outputs.run == 'true'
run: ls -lh dist/
shell: bash
- name: Test wheel with uv (pixi wheel_test)
if: steps.should_run.outputs.run == 'true' && runner.os != 'Windows'
env:
WHEEL_TEST_PYTHON_VERSION: cp${{ matrix.python-version == '3.12' && '312' || '313' }}
WHEEL_TEST_TYPE: ${{ matrix.wheel-type }}
run: pixi run -e ${{ matrix.pixi-environment }} wheel_test
- name: Test wheel with uv (Windows)
if: steps.should_run.outputs.run == 'true' && runner.os == 'Windows'
env:
WHEEL_TEST_PYTHON_VERSION: cp${{ matrix.python-version == '3.12' && '312' || '313' }}
WHEEL_TEST_TYPE: ${{ matrix.wheel-type }}
run: pixi run -e ${{ matrix.pixi-environment }} python scripts/test_wheel.py
publish_cpu:
name: Publish pymomentum-cpu to PyPI
needs: [build_cpu_wheels_linux, build_cpu_wheels, test_pip_wheels]
runs-on: ubuntu-latest
environment:
name: pypi-cpu
url: https://pypi.org/p/pymomentum-cpu
permissions:
id-token: write # IMPORTANT: mandatory for trusted publishing
# Only publish on tag push or manual workflow dispatch with publish=true
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true')
steps:
- name: Download CPU wheel artifacts
uses: actions/download-artifact@v7
with:
path: dist
pattern: wheels-cpu-*
merge-multiple: true
- name: List CPU distributions
run: ls -lh dist/
- name: Publish CPU to TestPyPI
if: |
github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.test_pypi == 'true'
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
- name: Publish CPU to PyPI
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.test_pypi != 'true')
uses: pypa/gh-action-pypi-publish@release/v1
publish_gpu:
name: Publish pymomentum-gpu to PyPI
needs: [build_gpu_wheels, test_pip_wheels]
runs-on: ubuntu-latest
environment:
name: pypi-gpu
url: https://pypi.org/p/pymomentum-gpu
permissions:
id-token: write # IMPORTANT: mandatory for trusted publishing
# Only publish on tag push or manual workflow dispatch with publish=true
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true')
steps:
- name: Download GPU wheel artifacts
uses: actions/download-artifact@v7
with:
path: dist
pattern: wheels-gpu-*
merge-multiple: true
- name: List GPU distributions
run: ls -lh dist/
- name: Publish GPU to TestPyPI
if: |
github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.test_pypi == 'true'
uses: pypa/gh-action-pypi-publish@release/v1
with:
repository-url: https://test.pypi.org/legacy/
- name: Publish GPU to PyPI
if: |
(github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.test_pypi != 'true')
uses: pypa/gh-action-pypi-publish@release/v1