Skip to content

Commit a9cf2e6

Browse files
CristianLarafacebook-github-bot
authored andcommitted
Introduce ExperimentStatus class (#4737)
Summary: # Summary Add `ExperimentStatus` enum with phases: DRAFT, INITIALIZATION, OPTIMIZATION, COMPLETED. **Backward compatibility:** The status field defaults to `None` for existing experiments, maintaining full backward compatibility. New experiments can optionally set status as needed. Reviewed By: mgarrard Differential Revision: D86801911
1 parent e984607 commit a9cf2e6

File tree

2 files changed

+140
-0
lines changed

2 files changed

+140
-0
lines changed

ax/core/experiment_status.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
#
4+
# This source code is licensed under the MIT license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# pyre-strict
8+
9+
from __future__ import annotations
10+
11+
from enum import Enum
12+
13+
14+
class ExperimentStatus(int, Enum):
15+
"""Enum of experiment status.
16+
17+
General lifecycle of an experiment is:::
18+
19+
DRAFT --> INITIALIZATION --> OPTIMIZATION --> COMPLETED
20+
21+
Experiment is marked as ``DRAFT`` immediately upon its creation when
22+
the experiment is still being configured (search space, optimization config, etc.).
23+
24+
Once the experiment is fully configured and begins initial exploration,
25+
it transitions to ``INITIALIZATION``. This is typically when the first trials
26+
are being generated to explore the search space.
27+
28+
After initial exploration completes (typically after some data has been collected),
29+
the experiment transitions to ``OPTIMIZATION``, where Bayesian optimization or
30+
other adaptive methods are used to find optimal configurations.
31+
32+
``COMPLETED`` indicates the experiment has successfully finished its objectives.
33+
34+
Note: This status tracks the high-level experiment lifecycle and is independent
35+
of individual trial statuses. An experiment in OPTIMIZATION status may have
36+
trials in various states (RUNNING, COMPLETED, FAILED, etc.).
37+
"""
38+
39+
DRAFT = 0
40+
INITIALIZATION = 1
41+
OPTIMIZATION = 2
42+
COMPLETED = 4
43+
44+
@property
45+
def is_active(self) -> bool:
46+
"""True if experiment is actively running trials."""
47+
return (
48+
self == ExperimentStatus.INITIALIZATION
49+
or self == ExperimentStatus.OPTIMIZATION
50+
)
51+
52+
@property
53+
def is_draft(self) -> bool:
54+
"""True if experiment is in draft phase."""
55+
return self == ExperimentStatus.DRAFT
56+
57+
@property
58+
def is_initialization(self) -> bool:
59+
"""True if experiment is in initialization phase."""
60+
return self == ExperimentStatus.INITIALIZATION
61+
62+
@property
63+
def is_optimization(self) -> bool:
64+
"""True if experiment is in optimization phase."""
65+
return self == ExperimentStatus.OPTIMIZATION
66+
67+
@property
68+
def is_completed(self) -> bool:
69+
"""True if experiment has successfully completed."""
70+
return self == ExperimentStatus.COMPLETED
71+
72+
def __format__(self, fmt: str) -> str:
73+
"""Define `__format__` to avoid pulling the `__format__` from the `int`
74+
mixin (since its better for statuses to show up as `DRAFT` than as
75+
just an int that is difficult to interpret).
76+
77+
E.g. experiment representation with the overridden method is:
78+
"Experiment(name='test', status=ExperimentStatus.DRAFT)".
79+
80+
Docs on enum formatting: https://docs.python.org/3/library/enum.html#others.
81+
"""
82+
return f"{self!s}"
83+
84+
def __repr__(self) -> str:
85+
return f"{self.__class__.__name__}.{self.name}"
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/usr/bin/env python3
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
#
4+
# This source code is licensed under the MIT license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
# pyre-strict
8+
9+
from ax.core.experiment_status import ExperimentStatus
10+
from ax.utils.common.testutils import TestCase
11+
12+
13+
class TestExperimentStatus(TestCase):
14+
"""Tests for the ExperimentStatus enum."""
15+
16+
def test_status_values(self) -> None:
17+
"""Test that status enum values are correctly defined."""
18+
self.assertEqual(ExperimentStatus.DRAFT.value, 0)
19+
self.assertEqual(ExperimentStatus.INITIALIZATION.value, 1)
20+
self.assertEqual(ExperimentStatus.OPTIMIZATION.value, 2)
21+
self.assertEqual(ExperimentStatus.COMPLETED.value, 4)
22+
23+
def test_is_active(self) -> None:
24+
"""Test the is_active property."""
25+
# Active statuses
26+
self.assertTrue(ExperimentStatus.INITIALIZATION.is_active)
27+
self.assertTrue(ExperimentStatus.OPTIMIZATION.is_active)
28+
29+
# Inactive statuses
30+
self.assertFalse(ExperimentStatus.DRAFT.is_active)
31+
self.assertFalse(ExperimentStatus.COMPLETED.is_active)
32+
33+
def test_individual_status_checks(self) -> None:
34+
"""Test individual status check properties."""
35+
self.assertTrue(ExperimentStatus.DRAFT.is_draft)
36+
self.assertFalse(ExperimentStatus.INITIALIZATION.is_draft)
37+
38+
self.assertTrue(ExperimentStatus.INITIALIZATION.is_initialization)
39+
self.assertFalse(ExperimentStatus.OPTIMIZATION.is_initialization)
40+
41+
self.assertTrue(ExperimentStatus.OPTIMIZATION.is_optimization)
42+
self.assertFalse(ExperimentStatus.COMPLETED.is_optimization)
43+
44+
self.assertTrue(ExperimentStatus.COMPLETED.is_completed)
45+
self.assertFalse(ExperimentStatus.DRAFT.is_completed)
46+
47+
def test_format_and_repr(self) -> None:
48+
"""Test __format__ and __repr__ methods."""
49+
status = ExperimentStatus.DRAFT
50+
self.assertEqual(f"{status}", "ExperimentStatus.DRAFT")
51+
self.assertEqual(repr(status), "ExperimentStatus.DRAFT")
52+
53+
status = ExperimentStatus.OPTIMIZATION
54+
self.assertEqual(f"{status}", "ExperimentStatus.OPTIMIZATION")
55+
self.assertEqual(repr(status), "ExperimentStatus.OPTIMIZATION")

0 commit comments

Comments
 (0)