FigURLCuration() populate issue

[denissemorales]

Describe the bug
I pulled the latest version of spyglass, and I get an error populating the FigURLCuration table (due to non-hashable objects (recording, sorting) being returned in the make_fetch function?)

My spyglass version: '0.5.5a2.dev38+ge345a020a.d20251229'
My datajoint version: 0.14.6

To Reproduce

figurl_curation_id = '9ad44512-f2ac-46d9-8ae2-0b52d7e2a19c'
sgs.FigURLCuration.populate({'figurl_curation_id': figurl_curation_id})

Error Stack

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[13], line 1
----> 1 sgs.FigURLCuration.populate({'figurl_curation_id': figurl_curation_id})

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/spyglass/utils/mixins/populate.py:98, in PopulateMixin.populate(self, *restrictions, **kwargs)
     96 if use_transact:  # Pass single-process populate to super
     97     kwargs["processes"] = processes
---> 98     return super().populate(*restrictions, **kwargs)
     99 else:  # No transaction protection, use bare make
    100     for key in keys:

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/datajoint/autopopulate.py:319, in AutoPopulate.populate(self, keys, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions)
    313 if processes == 1:
    314     for key in (
    315         tqdm(keys, desc=self.__class__.__name__)
    316         if display_progress
    317         else keys
    318     ):
--> 319         status = self._populate1(key, jobs, **populate_kwargs)
    320         if status is True:
    321             success_list.append(1)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/datajoint/autopopulate.py:402, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs)
    400 gen = make(dict(key), **(make_kwargs or {}))
    401 fetched_data = next(gen)
--> 402 fetch_hash = deepdiff.DeepHash(
    403     fetched_data, ignore_iterable_order=False
    404 )[fetched_data]
    405 computed_result = next(gen)  # perform the computation
    406 # fetch and insert inside a transaction

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:270, in DeepHash.__init__(self, obj, apply_hash, custom_operators, default_timezone, encodings, exclude_obj_callback, exclude_paths, exclude_regex_paths, exclude_types, hasher, hashes, ignore_encoding_errors, ignore_iterable_order, ignore_numeric_type_changes, ignore_private_variables, ignore_repetition, ignore_string_case, ignore_string_type_changes, ignore_type_in_groups, ignore_type_subclasses, ignore_uuid_types, include_paths, number_format_notation, number_to_string_func, parent, significant_digits, truncate_datetime, use_enum_value, **kwargs)
    267 self.ignore_iterable_order = ignore_iterable_order
    268 self.custom_operators = custom_operators
--> 270 self._hash(obj, parent=parent, parents_ids=frozenset({get_id(obj)}))
    272 if self.hashes[UNPROCESSED_KEY]:
    273     logger.warning("Can not hash the following items: {}.".format(self.hashes[UNPROCESSED_KEY]))

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:639, in DeepHash._hash(self, obj, parent, parents_ids)
    636     result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
    638 elif isinstance(obj, Iterable):
--> 639     result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
    641 elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
    642     result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:498, in DeepHash._prep_iterable(self, obj, parent, parents_ids)
    495     continue
    497 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 498 hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
    499 # counting repetitions
    500 result[hashed] += 1

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:646, in DeepHash._hash(self, obj, parent, parents_ids)
    644     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
    645 else:
--> 646     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
    648 if result is not_hashed:  # pragma: no cover
    649     self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:419, in DeepHash._prep_obj(self, obj, parent, parents_ids, is_namedtuple, is_pydantic_object)
    416     return (unprocessed, 0)
    417 obj = d
--> 419 result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
    420                                  print_as_attribute=True, original_type=original_type)
    421 result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
    422 return result, counts

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:465, in DeepHash._prep_dict(self, obj, parent, parents_ids, print_as_attribute, original_type)
    463     continue
    464 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 465 hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
    466 hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
    467 result.append(hashed)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:646, in DeepHash._hash(self, obj, parent, parents_ids)
    644     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
    645 else:
--> 646     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
    648 if result is not_hashed:  # pragma: no cover
    649     self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:419, in DeepHash._prep_obj(self, obj, parent, parents_ids, is_namedtuple, is_pydantic_object)
    416     return (unprocessed, 0)
    417 obj = d
--> 419 result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
    420                                  print_as_attribute=True, original_type=original_type)
    421 result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
    422 return result, counts

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:465, in DeepHash._prep_dict(self, obj, parent, parents_ids, print_as_attribute, original_type)
    463     continue
    464 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 465 hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
    466 hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
    467 result.append(hashed)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:639, in DeepHash._hash(self, obj, parent, parents_ids)
    636     result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
    638 elif isinstance(obj, Iterable):
--> 639     result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
    641 elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
    642     result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:498, in DeepHash._prep_iterable(self, obj, parent, parents_ids)
    495     continue
    497 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 498 hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
    499 # counting repetitions
    500 result[hashed] += 1

    [... skipping similar frames: DeepHash._hash at line 646 (1 times), DeepHash._prep_dict at line 465 (1 times), DeepHash._prep_obj at line 419 (1 times)]

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:639, in DeepHash._hash(self, obj, parent, parents_ids)
    636     result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
    638 elif isinstance(obj, Iterable):
--> 639     result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
    641 elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
    642     result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:498, in DeepHash._prep_iterable(self, obj, parent, parents_ids)
    495     continue
    497 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 498 hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
    499 # counting repetitions
    500 result[hashed] += 1

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:646, in DeepHash._hash(self, obj, parent, parents_ids)
    644     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
    645 else:
--> 646     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
    648 if result is not_hashed:  # pragma: no cover
    649     self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:419, in DeepHash._prep_obj(self, obj, parent, parents_ids, is_namedtuple, is_pydantic_object)
    416     return (unprocessed, 0)
    417 obj = d
--> 419 result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
    420                                  print_as_attribute=True, original_type=original_type)
    421 result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
    422 return result, counts

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:465, in DeepHash._prep_dict(self, obj, parent, parents_ids, print_as_attribute, original_type)
    463     continue
    464 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 465 hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
    466 hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
    467 result.append(hashed)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:620, in DeepHash._hash(self, obj, parent, parents_ids)
    617     result = str(obj.int)
    619 elif isinstance(obj, MutableMapping):
--> 620     result, counts = self._prep_dict(obj=obj, parent=parent, parents_ids=parents_ids)
    622 elif isinstance(obj, tuple):
    623     result, counts = self._prep_tuple(obj=obj, parent=parent, parents_ids=parents_ids)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:465, in DeepHash._prep_dict(self, obj, parent, parents_ids, print_as_attribute, original_type)
    463     continue
    464 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 465 hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
    466 hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
    467 result.append(hashed)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:646, in DeepHash._hash(self, obj, parent, parents_ids)
    644     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids, is_pydantic_object=True)
    645 else:
--> 646     result, counts = self._prep_obj(obj=obj, parent=parent, parents_ids=parents_ids)
    648 if result is not_hashed:  # pragma: no cover
    649     self.hashes[UNPROCESSED_KEY].append(obj)  # type: ignore

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:419, in DeepHash._prep_obj(self, obj, parent, parents_ids, is_namedtuple, is_pydantic_object)
    416     return (unprocessed, 0)
    417 obj = d
--> 419 result, counts = self._prep_dict(obj, parent=parent, parents_ids=parents_ids,
    420                                  print_as_attribute=True, original_type=original_type)
    421 result = "nt{}".format(result) if is_namedtuple else "obj{}".format(result)
    422 return result, counts

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:465, in DeepHash._prep_dict(self, obj, parent, parents_ids, print_as_attribute, original_type)
    463     continue
    464 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 465 hashed, count = self._hash(item, parent=key_in_report, parents_ids=parents_ids_added)
    466 hashed = KEY_TO_VAL_STR.format(key_hash, hashed)
    467 result.append(hashed)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:639, in DeepHash._hash(self, obj, parent, parents_ids)
    636     result, counts = self._prep_iterable(obj=gen(), parent=parent, parents_ids=parents_ids)
    638 elif isinstance(obj, Iterable):
--> 639     result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
    641 elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
    642     result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:498, in DeepHash._prep_iterable(self, obj, parent, parents_ids)
    495     continue
    497 parents_ids_added = add_to_frozen_set(parents_ids, item_id)
--> 498 hashed, count = self._hash(item, parent=new_parent, parents_ids=parents_ids_added)
    499 # counting repetitions
    500 result[hashed] += 1

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/deepdiff/deephash.py:641, in DeepHash._hash(self, obj, parent, parents_ids)
    638 elif isinstance(obj, Iterable):
    639     result, counts = self._prep_iterable(obj=obj, parent=parent, parents_ids=parents_ids)
--> 641 elif obj == BoolObj.TRUE or obj == BoolObj.FALSE:
    642     result = 'bool:true' if obj is BoolObj.TRUE else 'bool:false'
    643 elif isinstance(obj, PydanticBaseModel):

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/hdmf/utils.py:591, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    590 def func_call(*args, **kwargs):
--> 591     pargs = _check_args(args, kwargs)
    592     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass_1/lib/python3.10/site-packages/hdmf/utils.py:584, in docval.<locals>.dec.<locals>._check_args(args, kwargs)
    582     if parse_err:
    583         msg = '%s: %s' % (func.__qualname__, ', '.join(parse_err))
--> 584         raise ExceptionType(msg)
    586 return parsed['args']

TypeError: ElementIdentifiers.__eq__: incorrect type for 'other' (got 'BoolObj', expected 'Data, ndarray, list, tuple or int')

[samuelbray32]

What version of deepdiff do you have in your environment?

[CBroz1]

I was able to address the main issue here in #1505, but I encountered other issues with how sortingview 0.11 handles this specific sorting. Maybe your environment is different

My stack

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
File ~/wrk/spyglass/main/temp-1504.py:4
      1 from spyglass.spikesorting import v1 as sgs
      3 figurl_curation_id = "9ad44512-f2ac-46d9-8ae2-0b52d7e2a19c"
----> 4 sgs.FigURLCuration.populate({"figurl_curation_id": figurl_curation_id})

File ~/wrk/spyglass/main/src/spyglass/utils/mixins/populate.py:98, in PopulateMixin.populate(self, *restrictions, **kwargs)
     96 if use_transact:  # Pass single-process populate to super
     97     kwargs["processes"] = processes
---> 98     return super().populate(*restrictions, **kwargs)
     99 else:  # No transaction protection, use bare make
    100     for key in keys:

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/datajoint/autopopulate.py:319, in AutoPopulate.populate(self, keys, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions)
    313 if processes == 1:
    314     for key in (
    315         tqdm(keys, desc=self.__class__.__name__)
    316         if display_progress
    317         else keys
    318     ):
--> 319         status = self._populate1(key, jobs, **populate_kwargs)
    320         if status is True:
    321             success_list.append(1)

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/datajoint/autopopulate.py:405, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs)
    401 fetched_data = next(gen)
    402 fetch_hash = deepdiff.DeepHash(
    403     fetched_data, ignore_iterable_order=False
    404 )[fetched_data]
--> 405 computed_result = next(gen)  # perform the computation
    406 # fetch and insert inside a transaction
    407 self.connection.start_transaction()

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/datajoint/autopopulate.py:162, in AutoPopulate.make(self, key)
    159 # Step 2: If computed result is not passed in, compute the result
    160 if computed_result is None:
    161     # this is only executed in the first invocation
--> 162     computed_result = self.make_compute(key, *fetched_data)
    163     yield computed_result  # this is passed to the second invocation of make
    165 # Step 3: Insert the computed result into the current table.

File ~/wrk/spyglass/main/src/spyglass/spikesorting/v1/figurl_curation.py:194, in FigURLCuration.make_compute(self, key, object_id, sel_key, sorting_fpath, metrics_figurl, curation_uri, recording_label, sorting_label)
    189 unit_metrics = _reformat_metrics(metric_dict)
    191 # TODO: figure out a way to specify the similarity metrics
    192
    193 # Generate the figURL
--> 194 url = _generate_figurl(
    195     R=recording,
    196     S=sorting,
    197     initial_curation_uri=curation_uri,
    198     recording_label=recording_label,
    199     sorting_label=sorting_label,
    200     unit_metrics=unit_metrics,
    201 )
    203 return [url]

File ~/wrk/spyglass/main/src/spyglass/spikesorting/v1/figurl_curation.py:250, in _generate_figurl(R, S, initial_curation_uri, recording_label, sorting_label, unit_metrics, segment_duration_sec, snippet_ms_before, snippet_ms_after, max_num_snippets_per_segment, channel_neighborhood_size, raster_plot_subsample_max_firing_rate, spike_amplitudes_subsample_max_firing_rate)
    246 sorting = S
    248 sampling_frequency = recording.get_sampling_frequency()
--> 250 this_view = SpikeSortingView.create(
    251     recording=recording,
    252     sorting=sorting,
    253     segment_duration_sec=segment_duration_sec,
    254     snippet_len=(
    255         int(snippet_ms_before * sampling_frequency / 1000),
    256         int(snippet_ms_after * sampling_frequency / 1000),
    257     ),
    258     max_num_snippets_per_segment=max_num_snippets_per_segment,
    259     channel_neighborhood_size=channel_neighborhood_size,
    260 )
    262 # Assemble the views in a layout. Can be replaced with other layouts.
    263 raster_max_fire = raster_plot_subsample_max_firing_rate

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/sortingview/SpikeSortingView/SpikeSortingView.py:40, in SpikeSortingView.create(recording, sorting, segment_duration_sec, snippet_len, max_num_snippets_per_segment, channel_neighborhood_size, bandpass_filter, use_cache)
     29 @staticmethod
     30 def create(*,
     31     recording: si.BaseRecording,
   (...)
     38     use_cache: bool=True
     39 ):
---> 40     data_uri = prepare_spikesortingview_data(
     41         recording=recording,
     42         sorting=sorting,
     43         segment_duration_sec=segment_duration_sec,
     44         snippet_len=snippet_len,
     45         max_num_snippets_per_segment=max_num_snippets_per_segment,
     46         channel_neighborhood_size=channel_neighborhood_size,
     47         bandpass_filter=bandpass_filter,
     48         use_cache=use_cache
     49     )
     50     return SpikeSortingView(data_uri)

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/sortingview/SpikeSortingView/prepare_spikesortingview_data.py:25, in prepare_spikesortingview_data(recording, sorting, segment_duration_sec, snippet_len, max_num_snippets_per_segment, channel_neighborhood_size, bandpass_filter, use_cache)
     23 if use_cache:
     24     recording_object = get_recording_object(recording)
---> 25     sorting_object = get_sorting_object(sorting)
     26     cache_key_obj = {
     27         'type': 'spikesortingview_data',
     28         'version': 2,
   (...)
     34         'channel_neighborhood_size': channel_neighborhood_size
     35     }
     36     if bandpass_filter:

File ~/miniconda3/envs/spy/lib/python3.9/site-packages/sortingview/load_extractors/get_sorting_object.py:47, in get_sorting_object(sorting)
     40     sorting_object = {
     41         'sorting_format': 'npz',
     42         'data': {
     43             'npz_file_uri': npz_file_uri
     44         }
     45     }
     46 else:
---> 47     raise Exception('Unable to create sortingview object from sorting')
     48 setattr(sorting, 'sortingview_object', sorting_object)
     49 return sorting_object

Exception: Unable to create sortingview object from sorting