[ci] Add wanda anyscale image builds for release tests

andrew-anyscale · andrew-anyscale · commit 31e5688333db · 2026-01-29T16:45:18.000-08:00
This adds wanda-based builds for anyscale images used in release tests.

Changes:
- Add ray-anyscale-cpu/cuda.wanda.yaml for ray anyscale images
- Add ray-llm-anyscale-cuda.wanda.yaml for LLM images
- Add ray-ml-anyscale-cuda.wanda.yaml for ML images
- Add push_anyscale_image.py for pushing to ECR/GCP/Azure via crane
- Update release/build.rayci.yml with build and push steps
- Update gcloud_docker_login.sh to v550.0.0 with arch detection

Topic: anyscale-image
Relative: wanda-image-upload
Labels: draft
Signed-off-by: andrew &lt;andrew@anyscale.com&gt;
diff --git a/.buildkite/release/build.rayci.yml b/.buildkite/release/build.rayci.yml
@@ -1,5 +1,56 @@
 group: release build
 steps:
+  # Build ray core components (required for wheel builds)
+  - name: ray-core-build
+    label: "wanda: core binary parts py{{matrix}} (x86_64)"
+    wanda: ci/docker/ray-core.wanda.yaml
+    matrix:
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env_file: rayci.env
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+      HOSTTYPE: "x86_64"
+    tags: oss
+
+  - name: ray-dashboard-build
+    label: "wanda: dashboard"
+    wanda: ci/docker/ray-dashboard.wanda.yaml
+    env_file: rayci.env
+    env:
+      ARCH_SUFFIX: ""
+      HOSTTYPE: "x86_64"
+    tags: oss
+
+  - name: ray-java-build
+    label: "wanda: java build (x86_64)"
+    wanda: ci/docker/ray-java.wanda.yaml
+    tags: oss
+    env_file: rayci.env
+    env:
+      ARCH_SUFFIX: ""
+      HOSTTYPE: "x86_64"
+
+  - name: ray-wheel-build
+    label: "wanda: wheel py{{matrix}} (x86_64)"
+    wanda: ci/docker/ray-wheel.wanda.yaml
+    matrix:
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env_file: rayci.env
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+      HOSTTYPE: "x86_64"
+    tags: oss
+    depends_on:
+      - ray-core-build
+      - ray-java-build
+      - ray-dashboard-build
+
   - name: raycpubaseextra-testdeps
     label: "wanda: ray.py{{matrix}}.cpu.base-extra-testdeps"
     wanda: docker/base-extra-testdeps/cpu.wanda.yaml
@@ -65,26 +116,74 @@ steps:
     depends_on:
       - ray-mlcudabaseextra
 
-  - label: ":tapioca: build: ray py{{matrix.python}}-{{matrix.platform}} image for release tests"
+  # Build ray anyscale images using wanda (CPU)
+  - name: ray-anyscale-cpu-build
+    label: "wanda: ray-anyscale py{{matrix}} cpu"
+    wanda: ci/docker/ray-anyscale-cpu.wanda.yaml
+    env_file: rayci.env
+    matrix:
+      # This list should be kept in sync with the list of supported Python in
+      # release test suite
+      - "3.10"
+      - "3.11"
+      - "3.12"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+    tags:
+      - oss
+    depends_on:
+      - ray-wheel-build
+      - raycpubaseextra-testdeps
+
+  # Build ray anyscale images using wanda (CUDA)
+  - name: ray-anyscale-cuda-build
+    label: "wanda: ray-anyscale py{{matrix.python}} cu{{matrix.cuda}}"
+    wanda: ci/docker/ray-anyscale-cuda.wanda.yaml
+    env_file: rayci.env
+    matrix:
+      setup:
+        python:
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        cuda:
+          - "12.3.2-cudnn9"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      ARCH_SUFFIX: ""
+    tags:
+      - oss
+    depends_on:
+      - ray-wheel-build
+      - raycudabaseextra-testdeps
+
+  # Push anyscale images to ECR/GCP/Azure registries
+  - label: ":docker: push: ray-anyscale py{{matrix.python}} {{matrix.platform}}"
     key: anyscalebuild
     instance_type: release-medium
     mount_buildkite_agent: true
     tags:
       - oss
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- anyscale
-        --python-version {{matrix.python}} --platform {{matrix.platform}}
-        --image-type ray --upload
+      # Authenticate with GCP and Azure before pushing
+      - bash release/gcloud_docker_login.sh release/aws2gce_iam.json
+      - bash release/azure_docker_login.sh
+      - az acr login --name rayreleasetest
+      # PATH must include gcloud SDK for crane to use docker credential helper
+      - export PATH=$(pwd)/google-cloud-sdk/bin:$PATH &&
+        bazel run //ci/ray_ci/automation:push_anyscale_image --
+        --python-version {{matrix.python}}
+        --platform {{matrix.platform}}
+        --image-type ray
+        --upload
     depends_on:
-      - manylinux-x86_64
-      - forge
-      - raycpubaseextra-testdeps
-      - raycudabaseextra-testdeps
+      - ray-anyscale-cpu-build
+      - ray-anyscale-cuda-build
     matrix:
       setup:
         python:
-          # This list should be kept in sync with the list of supported Python in
-          # release test suite
           - "3.10"
           - "3.11"
           - "3.12"
@@ -93,35 +192,95 @@ steps:
           - cu12.3.2-cudnn9
           - cpu
 
-  - label: ":tapioca: build: ray-llm py{{matrix}} image for release tests"
+  # Build ray-llm anyscale image using wanda
+  - name: ray-llm-anyscale-cuda-build
+    label: "wanda: ray-llm-anyscale py{{matrix.python}} cu{{matrix.cuda}}"
+    wanda: ci/docker/ray-llm-anyscale-cuda.wanda.yaml
+    env_file: rayci.env
+    matrix:
+      setup:
+        python:
+          - "3.11"
+        cuda:
+          - "12.8.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      ARCH_SUFFIX: ""
+    tags:
+      - oss
+    depends_on:
+      - ray-wheel-build
+      - ray-llmbaseextra-testdeps
+
+  # Push ray-llm anyscale images to ECR/GCP/Azure registries
+  - label: ":docker: push: ray-llm-anyscale py{{matrix}} cu12.8.1-cudnn"
     key: anyscalellmbuild
     instance_type: release-medium
     mount_buildkite_agent: true
     tags:
       - oss
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- anyscale --python-version {{matrix}}
-        --platform cu12.8.1-cudnn --image-type ray-llm --upload
+      # Authenticate with GCP and Azure before pushing
+      - bash release/gcloud_docker_login.sh release/aws2gce_iam.json
+      - bash release/azure_docker_login.sh
+      - az acr login --name rayreleasetest
+      # PATH must include gcloud SDK for crane to use docker credential helper
+      - export PATH=$(pwd)/google-cloud-sdk/bin:$PATH &&
+        bazel run //ci/ray_ci/automation:push_anyscale_image --
+        --python-version {{matrix}}
+        --platform cu12.8.1-cudnn
+        --image-type ray-llm
+        --upload
     depends_on:
-      - manylinux-x86_64
-      - forge
-      - ray-llmbaseextra-testdeps
+      - ray-llm-anyscale-cuda-build
     matrix:
       - "3.11"
 
-  - label: ":tapioca: build: ray-ml py{{matrix}} image for release tests"
+  # Build ray-ml anyscale image using wanda
+  - name: ray-ml-anyscale-cuda-build
+    label: "wanda: ray-ml-anyscale py{{matrix.python}} cu{{matrix.cuda}}"
+    wanda: ci/docker/ray-ml-anyscale-cuda.wanda.yaml
+    env_file: rayci.env
+    matrix:
+      setup:
+        python:
+          # This list should be kept in sync with the list of supported Python in
+          # release test suite
+          - "3.10"
+        cuda:
+          - "12.1.1-cudnn8"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+      ARCH_SUFFIX: ""
+    tags:
+      - oss
+    depends_on:
+      - ray-wheel-build
+      - ray-mlcudabaseextra-testdeps
+
+  # Push ray-ml anyscale images to ECR/GCP/Azure registries
+  - label: ":docker: push: ray-ml-anyscale py{{matrix}} cu12.1.1-cudnn8"
     key: anyscalemlbuild
     instance_type: release-medium
     mount_buildkite_agent: true
     tags:
       - oss
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- anyscale --python-version {{matrix}}
-        --platform cu12.1.1-cudnn8 --image-type ray-ml --upload
+      # Authenticate with GCP and Azure before pushing
+      - bash release/gcloud_docker_login.sh release/aws2gce_iam.json
+      - bash release/azure_docker_login.sh
+      - az acr login --name rayreleasetest
+      # PATH must include gcloud SDK for crane to use docker credential helper
+      - export PATH=$(pwd)/google-cloud-sdk/bin:$PATH &&
+        bazel run //ci/ray_ci/automation:push_anyscale_image --
+        --python-version {{matrix}}
+        --platform cu12.1.1-cudnn8
+        --image-type ray-ml
+        --upload
     depends_on:
-      - manylinux-x86_64
-      - forge
-      - ray-mlcudabaseextra-testdeps
+      - ray-ml-anyscale-cuda-build
     matrix:
       # This list should be kept in sync with the list of supported Python in
       # release test suite
diff --git a/ci/docker/ray-anyscale-cpu.wanda.yaml b/ci/docker/ray-anyscale-cpu.wanda.yaml
@@ -0,0 +1,21 @@
+# Ray Anyscale CPU Image (for release tests)
+# Installs ray wheel into the CPU base-extra-testdeps image
+#
+# This produces the anyscale test image for CPU-only release tests.
+# Unlike the regular ray image, this uses base-extra-testdeps which
+# includes additional test dependencies.
+#
+name: "ray-anyscale-py$PYTHON_VERSION-cpu$ARCH_SUFFIX"
+disable_caching: true
+froms:
+  - "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cpu-base-extra-testdeps"  # CPU base with test deps
+  - "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"        # Ray wheel
+dockerfile: ci/docker/ray-image.Dockerfile
+build_args:
+  - PYTHON_VERSION
+  - ARCH_SUFFIX
+  - IMAGE_TYPE=ray
+  - PLATFORM=cpu
+  - BASE_VARIANT=base-extra-testdeps
+  - RAY_COMMIT=$BUILDKITE_COMMIT
+  - RAY_VERSION
diff --git a/ci/docker/ray-anyscale-cuda.wanda.yaml b/ci/docker/ray-anyscale-cuda.wanda.yaml
@@ -0,0 +1,21 @@
+# Ray Anyscale CUDA Image (for release tests)
+# Installs ray wheel into the CUDA base-extra-testdeps image
+#
+# This produces the anyscale test image for GPU release tests.
+# Unlike the regular ray image, this uses base-extra-testdeps which
+# includes additional test dependencies.
+#
+name: "ray-anyscale-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
+disable_caching: true
+froms:
+  - "cr.ray.io/rayproject/ray-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra-testdeps"  # CUDA base with test deps
+  - "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"                    # Ray wheel
+dockerfile: ci/docker/ray-image.Dockerfile
+build_args:
+  - PYTHON_VERSION
+  - ARCH_SUFFIX
+  - IMAGE_TYPE=ray
+  - BASE_VARIANT=base-extra-testdeps
+  - PLATFORM=cu$CUDA_VERSION
+  - RAY_COMMIT=$BUILDKITE_COMMIT
+  - RAY_VERSION
diff --git a/ci/docker/ray-llm-anyscale-cuda.wanda.yaml b/ci/docker/ray-llm-anyscale-cuda.wanda.yaml
@@ -0,0 +1,20 @@
+# Ray-LLM Anyscale CUDA Image (for release tests)
+# Installs ray wheel into the ray-llm CUDA base-extra-testdeps image
+#
+# This produces the anyscale test image for ray-llm GPU release tests.
+# Uses ray-llm base with extra test dependencies.
+#
+name: "ray-llm-anyscale-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
+disable_caching: true
+froms:
+  - "cr.ray.io/rayproject/ray-llm-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra-testdeps"  # ray-llm base with test deps
+  - "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"                        # Ray wheel
+dockerfile: ci/docker/ray-image.Dockerfile
+build_args:
+  - PYTHON_VERSION
+  - ARCH_SUFFIX
+  - BASE_VARIANT=base-extra-testdeps
+  - IMAGE_TYPE=ray-llm
+  - PLATFORM=cu$CUDA_VERSION
+  - RAY_COMMIT=$BUILDKITE_COMMIT
+  - RAY_VERSION
diff --git a/ci/docker/ray-ml-anyscale-cuda.wanda.yaml b/ci/docker/ray-ml-anyscale-cuda.wanda.yaml
@@ -0,0 +1,20 @@
+# Ray-ML Anyscale CUDA Image (for release tests)
+# Installs ray wheel into the ray-ml CUDA base-extra-testdeps image
+#
+# This produces the anyscale test image for ray-ml GPU release tests.
+# Uses ray-ml base with extra test dependencies.
+#
+name: "ray-ml-anyscale-py$PYTHON_VERSION-cu$CUDA_VERSION$ARCH_SUFFIX"
+disable_caching: true
+froms:
+  - "cr.ray.io/rayproject/ray-ml-py$PYTHON_VERSION-cu$CUDA_VERSION-base-extra-testdeps"  # ray-ml base with test deps
+  - "cr.ray.io/rayproject/ray-wheel-py$PYTHON_VERSION$ARCH_SUFFIX"                       # Ray wheel
+dockerfile: ci/docker/ray-image.Dockerfile
+build_args:
+  - PYTHON_VERSION
+  - ARCH_SUFFIX
+  - BASE_VARIANT=base-extra-testdeps
+  - IMAGE_TYPE=ray-ml
+  - PLATFORM=cu$CUDA_VERSION
+  - RAY_COMMIT=$BUILDKITE_COMMIT
+  - RAY_VERSION
diff --git a/ci/ray_ci/automation/BUILD.bazel b/ci/ray_ci/automation/BUILD.bazel
@@ -342,3 +342,30 @@ py_test(
         ci_require("pytest"),
     ],
 )
+
+py_binary(
+    name = "push_anyscale_image",
+    srcs = ["push_anyscale_image.py"],
+    exec_compatible_with = ["//bazel:py3"],
+    deps = [
+        ":crane_lib",
+        "//ci/ray_ci:ray_ci_lib",
+        "//release:ray_release",
+        ci_require("click"),
+    ],
+)
+
+py_test(
+    name = "test_push_anyscale_image",
+    size = "small",
+    srcs = ["test_push_anyscale_image.py"],
+    exec_compatible_with = ["//bazel:py3"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ":push_anyscale_image",
+        ci_require("pytest"),
+    ],
+)
diff --git a/ci/ray_ci/automation/push_anyscale_image.py b/ci/ray_ci/automation/push_anyscale_image.py
diff --git a/ci/ray_ci/automation/test_push_anyscale_image.py b/ci/ray_ci/automation/test_push_anyscale_image.py
