From dcfb1176535e770adf64198dd33c493609ac0440 Mon Sep 17 00:00:00 2001
From: Clay Rosenthal <clayros@amazon.com>
Date: Thu, 19 Feb 2026 14:38:58 -0800
Subject: [PATCH] Make nightly continue on error

---
 .github/workflows/nightly-training.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/nightly-training.yaml b/.github/workflows/nightly-training.yaml
index 4234a655..5b21714f 100644
--- a/.github/workflows/nightly-training.yaml
+++ b/.github/workflows/nightly-training.yaml
@@ -16,7 +16,9 @@ permissions:
 jobs:
   training:
     runs-on: codebuild-holosoma-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    continue-on-error: true
     strategy:
+      fail-fast: false # continue on one experiment failing
       matrix:
         experiment:
         - g1-29dof