gh-143120: pixi builds for free-threading and TSAN (#142872)

Co-authored-by: Isuru Fernando <isuruf@gmail.com>
Co-authored-by: Lucas Colley <lucas.colley8@gmail.com>

Author: 3 people

Tools/pixi-packages/README.md

@@ -1,36 +1,56 @@
 # CPython Pixi packages
 
-This directory contains definitions for [Pixi packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section)
-which can be built from the CPython source code.
+This directory contains definitions for [Pixi
+packages](https://pixi.sh/latest/reference/pixi_manifest/#the-package-section) which can
+be built from the CPython source code.
 
-Downstream developers can make use of these packages by adding them as Git dependencies in a
-[Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
+Downstream developers can make use of these packages by adding them as Git dependencies
+in a [Pixi workspace](https://pixi.sh/latest/first_workspace/), like:
 
 ```toml
 [dependencies]
-python = { git = "https://github.com/python/cpython", subdirectory = "Tools/pixi-packages/asan" }
+python.git = "https://github.com/python/cpython"
+python.subdirectory = "Tools/pixi-packages/asan"
 ```
 
 This is particularly useful when developers need to build CPython from source
-(for example, for an ASan-instrumented build), as it does not require any manual
+(for example, for an ASan or TSan-instrumented build), as it does not require any manual
 clone or build steps. Instead, Pixi will automatically handle both the build
 and installation of the package.
 
 Each package definition is contained in a subdirectory, but they share the build script
 `build.sh` in this directory. Currently defined package variants:
 
 - `default`
-- `asan`: ASan-instrumented build with `PYTHON_ASAN=1`
+- `freethreading`
+- `asan`: ASan-instrumented build
+- `tsan-freethreading`: TSan-instrumented free-threading build
 
 ## Maintenance
 
-- Keep the `version` fields in each `recipe.yaml` up to date with the Python version
-- Keep the dependency requirements up to date in each `recipe.yaml`
+- Keep the `abi_tag` and `version` fields in each `variants.yaml` up to date with the
+  Python version
 - Update `build.sh` for any breaking changes in the `configure` and `make` workflow
 
 ## Opportunities for future improvement
 
-- More package variants (such as TSan, UBSan)
+- More package variants (such as UBSan)
 - Support for Windows
-- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on https://github.com/prefix-dev/pixi/issues/4599
-- A workaround can be removed from the build script once https://github.com/prefix-dev/rattler-build/issues/2012 is resolved
+- Using a single `pixi.toml` and `recipe.yaml` for all package variants is blocked on
+  [pixi-build-backends#532](https://github.com/prefix-dev/pixi-build-backends/pull/532)
+  and [pixi#5248](https://github.com/prefix-dev/pixi/issues/5248)
+
+## Troubleshooting
+
+TSan builds may crash on Linux with
+```
+FATAL: ThreadSanitizer: unexpected memory mapping 0x7977bd072000-0x7977bd500000
+```
+To fix it, try reducing `mmap_rnd_bits`:
+
+```bash
+$ sudo sysctl vm.mmap_rnd_bits
+vm.mmap_rnd_bits = 32  # too high for TSan
+$ sudo sysctl vm.mmap_rnd_bits=28  # reduce it
+vm.mmap_rnd_bits = 28
+```

Tools/pixi-packages/asan/pixi.toml

@@ -1,6 +1,9 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 [workspace]
 channels = ["https://prefix.dev/conda-forge"]
-platforms = ["osx-arm64", "linux-64"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
 preview = ["pixi-build"]
 
 [package.build.backend]

Tools/pixi-packages/asan/recipe.yaml

@@ -1,61 +1,92 @@
+# NOTE: Please always only modify default/recipe.yaml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 context:
   # Keep up to date
-  version: "3.15"
+  freethreading_tag: ${{ "t" if "freethreading" in variant else "" }}
 
-package:
+recipe:
   name: python
-  version: ${{ version }}
 
 source:
   - path: ../../..
 
-build:
-  files:
-    exclude:
-      - "*.o"
-  script:
-    file: ../build.sh
-    env:
-      PYTHON_VARIANT: "asan"
-
-# derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
-requirements:
+outputs:
+- package:
+    name: python_abi
+    version: ${{ version }}
+  build:
+    string: "0_${{ abi_tag }}"
+  requirements:
+    run_constraints:
+      - python ${{ version }}.* *_${{ abi_tag }}
+
+- package:
+    name: python
+    version: ${{ version }}
   build:
-    - ${{ compiler('c') }}
-    - ${{ compiler('cxx') }}
-    - make
-    - pkg-config
-    # configure script looks for llvm-ar for lto
-    - if: osx
-      then:
-        - llvm-tools
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - binutils_impl_${{ target_platform }}
-    - clang-19
-    - llvm-tools-19
-
-  host:
-    - bzip2
-    - sqlite
-    - liblzma-devel
-    - zlib
-    - zstd
-    - openssl
-    - readline
-    - tk
-    # These two are just to get the headers needed for tk.h, but is unused
-    - xorg-libx11
-    - xorg-xorgproto
-    - ncurses
-    - libffi
-    - if: linux
-      then:
-        - ld_impl_${{ target_platform }}
-        - libuuid
-    - libmpdec-devel
-    - expat
+    string: "0_${{ abi_tag }}"
+    files:
+      exclude:
+        - "*.o"
+    script:
+      file: ../build.sh
+      env:
+        PYTHON_VARIANT: ${{ variant }}
+    python:
+      site_packages_path: "lib/python${{ version }}${{ freethreading_tag }}/site-packages"
+
+  # derived from https://github.com/conda-forge/python-feedstock/blob/main/recipe/meta.yaml
+  requirements:
+    build:
+      - ${{ compiler('c') }}
+      - ${{ compiler('cxx') }}
+      # Note that we are not using stdlib arguments which means the packages
+      # are built for the build settings and are not relocatable to a different
+      # machine that has a older system version. (eg: macOS/glibc version)
+      - make
+      - pkg-config
+      # configure script looks for llvm-ar for lto
+      - if: osx
+        then:
+          - llvm-tools
+
+    host:
+      - bzip2
+      - sqlite
+      - liblzma-devel
+      - zlib
+      - zstd
+      - openssl
+      - readline
+      - tk
+      # These two are just to get the headers needed for tk.h, but is unused
+      - xorg-libx11
+      - xorg-xorgproto
+      - ncurses
+      - libffi
+      - if: linux
+        then:
+          - libuuid
+      - libmpdec-devel
+      - expat
+      - if: linux and "san" in variant
+        then:
+          - libsanitizer
+      - if: osx and "san" in variant
+        then:
+          - libcompiler-rt
+
+    ignore_run_exports:
+      from_package:
+        - xorg-libx11
+        - xorg-xorgproto
+
+    run_exports:
+      noarch:
+        - python
+      weak:
+        - python_abi ${{ version }}.* *_${{ abi_tag }}
 
 about:
   homepage: https://www.python.org/

Tools/pixi-packages/asan/variants.yaml

@@ -0,0 +1,6 @@
+variant:
+  - asan
+abi_tag:
+  - asan_cp315
+version:
+  - 3.15

Tools/pixi-packages/build.sh

@@ -1,17 +1,36 @@
 #!/bin/bash
 
-if [[ "${PYTHON_VARIANT}" == "asan" ]]; then
-    echo "BUILD TYPE: ASAN"
-    BUILD_DIR="../build_asan"
+echo "PYTHON_VARIANT: ${PYTHON_VARIANT}"
+
+if [[ "${PYTHON_VARIANT}" == "freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil"
+elif [[ "${PYTHON_VARIANT}" == "asan" ]]; then
     CONFIGURE_EXTRA="--with-address-sanitizer"
-    export PYTHON_ASAN="1"
     export ASAN_OPTIONS="strict_init_order=true"
-else
-    echo "BUILD TYPE: DEFAULT"
-    BUILD_DIR="../build"
+elif [[ "${PYTHON_VARIANT}" == "tsan-freethreading" ]]; then
+    CONFIGURE_EXTRA="--disable-gil --with-thread-sanitizer"
+    export TSAN_OPTIONS="suppressions=${SRC_DIR}/Tools/tsan/suppressions_free_threading.txt"
+elif [[ "${PYTHON_VARIANT}" == "default" ]]; then
     CONFIGURE_EXTRA=""
+else
+    echo "Unknown PYTHON_VARIANT: ${PYTHON_VARIANT}"
+    exit 1
 fi
 
+# rattler-build by default set a target of 10.9
+# override it to at least 10.12
+case ${MACOSX_DEPLOYMENT_TARGET:-10.12} in
+    10.12|10.13|10.14|10.15|10.16)
+        ;;
+    10.*)
+        export CPPFLAGS=${CPPFLAGS/-mmacosx-version-min=${MACOSX_DEPLOYMENT_TARGET}/-mmacosx-version-min=10.12}
+        export MACOSX_DEPLOYMENT_TARGET=10.12
+        ;;
+    *)
+        ;;
+esac
+
+BUILD_DIR="../build_${PYTHON_VARIANT}"
 mkdir -p "${BUILD_DIR}"
 cd "${BUILD_DIR}"
 
@@ -23,15 +42,11 @@ else
         --oldincludedir="${BUILD_PREFIX}/${HOST}/sysroot/usr/include" \
         --enable-shared \
         --srcdir="${SRC_DIR}" \
+        --with-system-expat \
         ${CONFIGURE_EXTRA}
 fi
 
 touch configure-done
 
 make -j"${CPU_COUNT}" install
 ln -sf "${PREFIX}/bin/python3" "${PREFIX}/bin/python"
-
-# https://github.com/prefix-dev/rattler-build/issues/2012
-if [[ ${OSTYPE} == "darwin"* ]]; then
-    cp "${BUILD_PREFIX}/lib/clang/21/lib/darwin/libclang_rt.asan_osx_dynamic.dylib" "${PREFIX}/lib/libclang_rt.asan_osx_dynamic.dylib"
-fi

Tools/pixi-packages/clone-recipe.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Please always only modify default/recipe.yaml and default/pixi.toml and then run this
+# script to propagate the changes to the other variants.
+set -o errexit
+cd "$(dirname "$0")"
+
+for variant in asan freethreading tsan-freethreading; do
+    cp -av default/recipe.yaml default/pixi.toml ${variant}/
+done

Tools/pixi-packages/default/pixi.toml

@@ -1,6 +1,9 @@
+# NOTE: Please always only modify default/pixi.toml and then run clone-recipe.sh to
+# propagate the changes to the other variants.
+
 [workspace]
 channels = ["https://prefix.dev/conda-forge"]
-platforms = ["osx-arm64", "linux-64"]
+platforms = ["linux-64", "linux-aarch64", "osx-64", "osx-arm64"]
 preview = ["pixi-build"]
 
 [package.build.backend]