Merge branch 'main' of github.com:SpikeInterface/spikeinterface into expose-zarr-compression

Author: alejoe91

src/spikeinterface/core/sorting_tools.py

@@ -197,17 +197,23 @@ def random_spikes_selection(
         cum_sizes = np.cumsum([0] + [s.size for s in spikes])
 
         # this fast when numba
-        spike_indices = spike_vector_to_indices(spikes, sorting.unit_ids)
+        spike_indices = spike_vector_to_indices(spikes, sorting.unit_ids, absolute_index=False)
 
         random_spikes_indices = []
         for unit_index, unit_id in enumerate(sorting.unit_ids):
             all_unit_indices = []
             for segment_index in range(sorting.get_num_segments()):
-                inds_in_seg = spike_indices[segment_index][unit_id] + cum_sizes[segment_index]
+                # this is local index
+                inds_in_seg = spike_indices[segment_index][unit_id]
                 if margin_size is not None:
-                    inds_in_seg = inds_in_seg[inds_in_seg >= margin_size]
-                    inds_in_seg = inds_in_seg[inds_in_seg < (num_samples[segment_index] - margin_size)]
-                all_unit_indices.append(inds_in_seg)
+                    local_spikes = spikes[segment_index][inds_in_seg]
+                    mask = (local_spikes["sample_index"] >= margin_size) & (
+                        local_spikes["sample_index"] < (num_samples[segment_index] - margin_size)
+                    )
+                    inds_in_seg = inds_in_seg[mask]
+                # go back to absolut index
+                inds_in_seg_abs = inds_in_seg + cum_sizes[segment_index]
+                all_unit_indices.append(inds_in_seg_abs)
             all_unit_indices = np.concatenate(all_unit_indices)
             selected_unit_indices = rng.choice(
                 all_unit_indices, size=min(max_spikes_per_unit, all_unit_indices.size), replace=False, shuffle=False

src/spikeinterface/core/sortinganalyzer.py

@@ -2062,7 +2062,8 @@ def load_data(self):
                     continue
                 ext_data_name = ext_data_file.stem
                 if ext_data_file.suffix == ".json":
-                    ext_data = json.load(ext_data_file.open("r"))
+                    with ext_data_file.open("r") as f:
+                        ext_data = json.load(f)
                 elif ext_data_file.suffix == ".npy":
                     # The lazy loading of an extension is complicated because if we compute again
                     # and have a link to the old buffer on windows then it fails
@@ -2074,7 +2075,8 @@ def load_data(self):
 
                     ext_data = pd.read_csv(ext_data_file, index_col=0)
                 elif ext_data_file.suffix == ".pkl":
-                    ext_data = pickle.load(ext_data_file.open("rb"))
+                    with ext_data_file.open("rb") as f:
+                        ext_data = pickle.load(f)
                 else:
                     continue
                 self.data[ext_data_name] = ext_data

src/spikeinterface/core/tests/test_sorting_tools.py

@@ -162,8 +162,8 @@ def test_generate_unit_ids_for_merge_group():
 if __name__ == "__main__":
     # test_spike_vector_to_spike_trains()
     # test_spike_vector_to_indices()
-    # test_random_spikes_selection()
+    test_random_spikes_selection()
 
-    test_apply_merges_to_sorting()
-    test_get_ids_after_merging()
+    # test_apply_merges_to_sorting()
+    # test_get_ids_after_merging()
     # test_generate_unit_ids_for_merge_group()

src/spikeinterface/qualitymetrics/quality_metric_calculator.py

@@ -234,7 +234,8 @@ def _run(self, verbose=False, **job_kwargs):
         )
 
         existing_metrics = []
-        qm_extension = self.sorting_analyzer.get_extension("quality_metrics")
+        # here we get in the loaded via the dict only (to avoid full loading from disk after params reset)
+        qm_extension = self.sorting_analyzer.extensions.get("quality_metrics", None)
         if (
             delete_existing_metrics is False
             and qm_extension is not None

src/spikeinterface/sorters/utils/shellscript.py

@@ -86,15 +86,15 @@ def start(self) -> None:
         if self._verbose:
             print("RUNNING SHELL SCRIPT: " + cmd)
         self._start_time = time.time()
+        encoding = sys.getdefaultencoding()
         self._process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding=encoding
         )
         with open(script_log_path, "w+") as script_log_file:
             for line in self._process.stdout:
                 script_log_file.write(line)
-                if (
-                    self._verbose
-                ):  # Print onto console depending on the verbose property passed on from the sorter class
+                if self._verbose:
+                    # Print onto console depending on the verbose property passed on from the sorter class
                     print(line)
 
     def wait(self, timeout=None) -> Optional[int]: