[detectors] Complete timecode migration

Complete migrating all existing detectors and `StatsManager` to use timecodes instead of frame numbers.

In some cases (e.g. `ThresholdDetector`), we still use frame numbers pending conversion of the algorithm.

Author: Breakthrough

scenedetect/detectors/adaptive_detector.py

@@ -81,7 +81,7 @@ def __init__(
             kernel_size=kernel_size,
         )
 
-        # TODO: Turn these options into properties.
+        # TODO: Turn these public options into properties.
         self.min_scene_len = min_scene_len
         self.adaptive_threshold = adaptive_threshold
         self.min_content_val = min_content_val
@@ -90,41 +90,21 @@ def __init__(
         self._adaptive_ratio_key = AdaptiveDetector.ADAPTIVE_RATIO_KEY_TEMPLATE.format(
             window_width=window_width, luma_only="" if not luma_only else "_lum"
         )
-        self._first_frame_num = None
-
-        # NOTE: This must be different than `self._last_scene_cut` which is used by the base class.
-        self._last_cut: ty.Optional[int] = None
-
-        self._buffer = []
+        self._buffer: ty.List[ty.Tuple[FrameTimecode, float]] = []
+        # NOTE: The name of last cut is different from `self._last_scene_cut` from our base class,
+        # and serves a different purpose!
+        self._last_cut: ty.Optional[FrameTimecode] = None
 
     @property
     def event_buffer_length(self) -> int:
-        """Number of frames any detected cuts will be behind the current frame due to buffering."""
         return self.window_width
 
     def get_metrics(self) -> ty.List[str]:
-        """Combines base ContentDetector metric keys with the AdaptiveDetector one."""
         return super().get_metrics() + [self._adaptive_ratio_key]
 
-    def stats_manager_required(self) -> bool:
-        """Not required for AdaptiveDetector."""
-        return False
-
     def process_frame(
-        self, timecode: FrameTimecode, frame_img: ty.Optional[np.ndarray]
-    ) -> ty.List[int]:
-        """Process the next frame. `frame_num` is assumed to be sequential.
-
-        Args:
-            frame_num (int): Frame number of frame that is being passed. Can start from any value
-                but must remain sequential.
-            frame_img (numpy.ndarray or None): Video frame corresponding to `frame_img`.
-
-        Returns:
-           ty.List[int]: List of frames where scene cuts have been detected. There may be 0
-            or more frames in the list, and not necessarily the same as frame_num.
-        """
-
+        self, timecode: FrameTimecode, frame_img: np.ndarray
+    ) -> ty.List[FrameTimecode]:
         # TODO(#283): Merge this with ContentDetector and turn it on by default.
 
         super().process_frame(timecode=timecode, frame_img=frame_img)
@@ -138,7 +118,7 @@ def process_frame(
         if not len(self._buffer) >= required_frames:
             return []
         self._buffer = self._buffer[-required_frames:]
-        (target_frame, target_score) = self._buffer[self.window_width]
+        (target_timecode, target_score) = self._buffer[self.window_width]
         average_window_score = sum(
             score for i, (_frame, score) in enumerate(self._buffer) if i != self.window_width
         ) / (2.0 * self.window_width)
@@ -152,7 +132,9 @@ def process_frame(
             # if we would have divided by zero, set adaptive_ratio to the max (255.0)
             adaptive_ratio = 255.0
         if self.stats_manager is not None:
-            self.stats_manager.set_metrics(target_frame, {self._adaptive_ratio_key: adaptive_ratio})
+            self.stats_manager.set_metrics(
+                target_timecode, {self._adaptive_ratio_key: adaptive_ratio}
+            )
 
         # Check to see if adaptive_ratio exceeds the adaptive_threshold as well as there
         # being a large enough content_val to trigger a cut
@@ -161,21 +143,6 @@ def process_frame(
         )
         min_length_met: bool = (timecode - self._last_cut) >= self.min_scene_len
         if threshold_met and min_length_met:
-            self._last_cut = target_frame
-            return [target_frame]
-        return []
-
-    def get_content_val(self, frame_num: int) -> ty.Optional[float]:
-        """Returns the average content change for a frame."""
-        # TODO(v0.7): Add DeprecationWarning that `get_content_val` will be removed in v0.7.
-        logger.error(
-            "get_content_val is deprecated and will be removed. Lookup the value"
-            " using a StatsManager with ContentDetector.FRAME_SCORE_KEY."
-        )
-        if self.stats_manager is not None:
-            return self.stats_manager.get_metrics(frame_num, [ContentDetector.FRAME_SCORE_KEY])[0]
-        return 0.0
-
-    def post_process(self, _unused_frame_num: int):
-        """Not required for AdaptiveDetector."""
+            self._last_cut = target_timecode
+            return [target_timecode]
         return []

scenedetect/detectors/content_detector.py

@@ -179,7 +179,7 @@ def _calculate_frame_score(self, timecode: FrameTimecode, frame_img: numpy.ndarr
         if self.stats_manager is not None:
             metrics = {self.FRAME_SCORE_KEY: frame_score}
             metrics.update(score_components._asdict())
-            self.stats_manager.set_metrics(timecode.frame_num, metrics)
+            self.stats_manager.set_metrics(timecode, metrics)
 
         # Store all data required to calculate the next frame's score.
         self._last_frame = ContentDetector._FrameData(hue, sat, lum, edges)

scenedetect/detectors/hash_detector.py

@@ -1,43 +1,27 @@
 #
-#         PySceneDetect: Python-Based Video Scene Detector
-#   ---------------------------------------------------------------
-#     [  Site: http://www.bcastell.com/projects/PySceneDetect/   ]
-#     [  Github: https://github.com/Breakthrough/PySceneDetect/  ]
-#     [  Documentation: http://pyscenedetect.readthedocs.org/    ]
+#            PySceneDetect: Python-Based Video Scene Detector
+#   -------------------------------------------------------------------
+#     [  Site:    https://scenedetect.com                           ]
+#     [  Docs:    https://scenedetect.com/docs/                     ]
+#     [  Github:  https://github.com/Breakthrough/PySceneDetect/    ]
 #
 # Copyright (C) 2014-2022 Brandon Castellano <http://www.bcastell.com>.
+# PySceneDetect is licensed under the BSD 3-Clause License; see the
+# included LICENSE file, or visit one of the above pages for details.
 #
-# PySceneDetect is licensed under the BSD 3-Clause License; see the included
-# LICENSE file, or visit one of the following pages for details:
-#  - https://github.com/Breakthrough/PySceneDetect/
-#  - http://www.bcastell.com/projects/PySceneDetect/
-#
-# This software uses Numpy, OpenCV, click, tqdm, simpletable, and pytest.
-# See the included LICENSE files or one of the above URLs for more information.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-# AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
-# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-#
-"""``scenedetect.detectors.hash_detector`` Module
+""":py:class:`HashDetector` calculates a hash for each frame of a video using a perceptual
+hashing algorithm. The differences (distance) in hash value between frames is calculated.
+If this difference exceeds a set threshold, a scene cut is triggered.
 
-This module implements the :py:class:`HashDetector`, which calculates a hash
-value for each from of a video using a perceptual hashing algorithm. Then, the
-differences in hash value between frames is calculated. If this difference
-exceeds a set threshold, a scene cut is triggered.
-
-This detector is available from the command-line interface by using the
-`detect-hash` command.
+This detector is available from the command-line interface by using the `detect-hash` command.
 """
 
-# Third-Party Library Imports
+import typing as ty
+
 import cv2
 import numpy
 
-# PySceneDetect Library Imports
+from scenedetect.common import FrameTimecode
 from scenedetect.detector import SceneDetector
 
 
@@ -74,15 +58,17 @@ def __init__(
         self._size = size
         self._size_sq = float(size * size)
         self._factor = lowpass
-        self._last_frame = None
-        self._last_scene_cut = None
+        self._last_frame: numpy.ndarray = None
+        self._last_scene_cut: FrameTimecode = None
         self._last_hash = numpy.array([])
         self._metric_key = f"hash_dist [size={self._size} lowpass={self._factor}]"
 
     def get_metrics(self):
         return [self._metric_key]
 
-    def process_frame(self, frame_num: int, frame_img: numpy.ndarray):
+    def process_frame(
+        self, timecode: FrameTimecode, frame_img: numpy.ndarray
+    ) -> ty.List[FrameTimecode]:
         """Similar to ContentDetector, but using a perceptual hashing algorithm
         to calculate a hash for each frame and then calculate a hash difference
         frame to frame."""
@@ -91,7 +77,7 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray):
 
         # Initialize last scene cut point at the beginning of the frames of interest.
         if self._last_scene_cut is None:
-            self._last_scene_cut = frame_num
+            self._last_scene_cut = timecode
 
         # We can only start detecting once we have a frame to compare with.
         if self._last_frame is not None:
@@ -115,17 +101,17 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray):
             hash_dist_norm = hash_dist / self._size_sq
 
             if self.stats_manager is not None:
-                self.stats_manager.set_metrics(frame_num, {self._metric_key: hash_dist_norm})
+                self.stats_manager.set_metrics(timecode, {self._metric_key: hash_dist_norm})
 
             self._last_hash = curr_hash
 
             # We consider any frame over the threshold a new scene, but only if
             # the minimum scene length has been reached (otherwise it is ignored).
             if hash_dist_norm >= self._threshold and (
-                (frame_num - self._last_scene_cut) >= self._min_scene_len
+                (timecode - self._last_scene_cut) >= self._min_scene_len
             ):
-                cut_list.append(frame_num)
-                self._last_scene_cut = frame_num
+                cut_list.append(timecode)
+                self._last_scene_cut = timecode
 
         self._last_frame = frame_img.copy()
 

scenedetect/detectors/histogram_detector.py

@@ -20,7 +20,7 @@
 import cv2
 import numpy
 
-# PySceneDetect Library Imports
+from scenedetect.common import FrameTimecode
 from scenedetect.detector import SceneDetector
 
 
@@ -48,10 +48,10 @@ def __init__(self, threshold: float = 0.05, bins: int = 256, min_scene_len: int
         self._bins = bins
         self._min_scene_len = min_scene_len
         self._last_hist = None
-        self._last_scene_cut = None
+        self._last_cut = None
         self._metric_key = f"hist_diff [bins={self._bins}]"
 
-    def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int]:
+    def process_frame(self, timecode: FrameTimecode, frame_img: numpy.ndarray) -> ty.List[int]:
         """Computes the histogram of the luma channel of the frame image and compares it with the
         histogram of the luma channel of the previous frame. If the difference between the histograms
         exceeds the threshold, a scene cut is detected.
@@ -77,8 +77,8 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int
             raise ValueError("Image must have three color channels for HistogramDetector")
 
         # Initialize last scene cut point at the beginning of the frames of interest.
-        if not self._last_scene_cut:
-            self._last_scene_cut = frame_num
+        if not self._last_cut:
+            self._last_cut = timecode
 
         hist = self.calculate_histogram(frame_img, bins=self._bins)
 
@@ -98,14 +98,14 @@ def process_frame(self, frame_num: int, frame_img: numpy.ndarray) -> ty.List[int
             # Example: If `_threshold` is set to 0.8, it implies that only changes resulting in a correlation
             # less than 0.8 between histograms will be considered significant enough to denote a scene change.
             if hist_diff <= self._threshold and (
-                (frame_num - self._last_scene_cut) >= self._min_scene_len
+                (timecode - self._last_cut) >= self._min_scene_len
             ):
-                cut_list.append(frame_num)
-                self._last_scene_cut = frame_num
+                cut_list.append(timecode)
+                self._last_cut = timecode
 
             # Save stats to a StatsManager if it is being used
             if self.stats_manager is not None:
-                self.stats_manager.set_metrics(frame_num, {self._metric_key: hist_diff})
+                self.stats_manager.set_metrics(timecode, {self._metric_key: hist_diff})
 
         self._last_hist = hist
 

scenedetect/detectors/threshold_detector.py

@@ -93,7 +93,9 @@ def __init__(
     def get_metrics(self) -> ty.List[str]:
         return self._metric_keys
 
-    def process_frame(self, timecode: FrameTimecode, frame_img: numpy.ndarray) -> ty.List[int]:
+    def process_frame(
+        self, timecode: FrameTimecode, frame_img: numpy.ndarray
+    ) -> ty.List[FrameTimecode]:
         """Process the next frame. `frame_num` is assumed to be sequential.
 
         Args:
@@ -105,7 +107,8 @@ def process_frame(self, timecode: FrameTimecode, frame_img: numpy.ndarray) -> ty
             ty.List[int]: List of frames where scene cuts have been detected. There may be 0
             or more frames in the list, and not necessarily the same as frame_num.
         """
-        # TODO(v0.7): We might need to consider PTS here instead.
+        # TODO(v0.7): We need to consider PTS here instead. The methods below using frame numbers
+        # won't work for variable framerates.
         frame_num = timecode.frame_num
 
         # Initialize last scene cut point at the beginning of the frames of interest.
@@ -130,7 +133,7 @@ def process_frame(self, timecode: FrameTimecode, frame_img: numpy.ndarray) -> ty
         else:
             frame_avg = numpy.mean(frame_img)
             if self.stats_manager is not None:
-                self.stats_manager.set_metrics(frame_num, {self._metric_keys[0]: frame_avg})
+                self.stats_manager.set_metrics(timecode, {self._metric_keys[0]: frame_avg})
 
         if self.processed_frame:
             if self.last_fade["type"] == "in" and (
@@ -166,7 +169,7 @@ def process_frame(self, timecode: FrameTimecode, frame_img: numpy.ndarray) -> ty
         self.processed_frame = True
         return [FrameTimecode(cut, fps=timecode) for cut in cuts]
 
-    def post_process(self, timecode: FrameTimecode):
+    def post_process(self, timecode: FrameTimecode) -> ty.List[FrameTimecode]:
         """Writes a final scene cut if the last detected fade was a fade-out.
 
         Only writes the scene cut if add_final_scene is true, and the last fade