Set trials to abandon on _mark_err_trial_status (facebook#4779)

Summary:
Pull Request resolved: facebook#4779

When a metric used in the OptimizationConfig fails to fetch, mark the trial as ABANDONED rather than FAILED so that it is less likely to be tried again.

Reviewed By: mpolson64

Differential Revision: D90884185

fbshipit-source-id: c23af50d3f7633ef82cfc2c8729b7d21be830c5a

Author: bernardbeckerman

ax/orchestration/orchestrator.py

@@ -2121,9 +2121,10 @@ def _mark_err_trial_status(
         metric_name: str | None = None,
         metric_fetch_e: MetricFetchE | None = None,
     ) -> TrialStatus:
-        trial.mark_failed(unsafe=True)
-
-        return TrialStatus.FAILED
+        trial.mark_abandoned(
+            reason=metric_fetch_e.message if metric_fetch_e else None, unsafe=True
+        )
+        return TrialStatus.ABANDONED
 
     def _get_failure_rate_exceeded_error(
         self,

ax/orchestration/tests/test_orchestrator.py

@@ -1857,14 +1857,16 @@ def test_fetch_and_process_trials_data_results_failed_objective(self) -> None:
             any(
                 re.search(
                     r"Because (branin|m1) is an objective, marking trial 0 as "
-                    "TrialStatus.FAILED",
+                    "TrialStatus.ABANDONED",
                     warning,
                 )
                 is not None
                 for warning in lg.output
             )
         )
-        self.assertEqual(orchestrator.experiment.trials[0].status, TrialStatus.FAILED)
+        self.assertEqual(
+            orchestrator.experiment.trials[0].status, TrialStatus.ABANDONED
+        )
 
     def test_fetch_and_process_trials_data_results_failed_objective_but_recoverable(
         self,
@@ -1957,14 +1959,16 @@ def test_fetch_and_process_trials_data_results_failed_objective_not_recoverable(
             any(
                 re.search(
                     r"Because (branin|m1) is an objective, marking trial 0 as "
-                    "TrialStatus.FAILED",
+                    "TrialStatus.ABANDONED",
                     warning,
                 )
                 is not None
                 for warning in lg.output
             )
         )
-        self.assertEqual(orchestrator.experiment.trials[0].status, TrialStatus.FAILED)
+        self.assertEqual(
+            orchestrator.experiment.trials[0].status, TrialStatus.ABANDONED
+        )
 
     def test_should_consider_optimization_complete(self) -> None:
         # Tests non-GSS parts of the completion criterion.