diff --git a/.github/workflows/nightly-training.yaml b/.github/workflows/nightly-training.yaml
index 4234a655..5b21714f 100644
--- a/.github/workflows/nightly-training.yaml
+++ b/.github/workflows/nightly-training.yaml
@@ -16,7 +16,9 @@ permissions:
 jobs:
   training:
     runs-on: codebuild-holosoma-gpu-build-${{ github.run_id }}-${{ github.run_attempt }}
+    continue-on-error: true
     strategy:
+      fail-fast: false # continue on one experiment failing
       matrix:
         experiment:
         - g1-29dof