diff --git a/src/damaris/data/DataPool.py b/src/damaris/data/DataPool.py index c55fc1e..8bf6a39 100644 --- a/src/damaris/data/DataPool.py +++ b/src/damaris/data/DataPool.py @@ -47,20 +47,20 @@ class DataPool(collections.abc.MutableMapping): def __init__(self): self.__mydict={} - self.__dictlock=threading.Lock() - self.__registered_listeners=[] + self._dictlock=threading.RLock() + self._registered_listeners=[] self.__dirty_keys=set() def __getitem__(self, name): try: - self.__dictlock.acquire() + self._dictlock.acquire() return self.__mydict[name] finally: - self.__dictlock.release() + self._dictlock.release() def __setitem__(self, name, value): try: - self.__dictlock.acquire() + self._dictlock.acquire() if name in self.__mydict: e=DataPool.Event(DataPool.Event.updated_value,name,self) else: @@ -68,47 +68,47 @@ class DataPool(collections.abc.MutableMapping): self.__mydict[name]=value self.__dirty_keys.add(name) finally: - self.__dictlock.release() + self._dictlock.release() self.__send_event(e) def __delitem__(self, name): try: - self.__dictlock.acquire() + self._dictlock.acquire() del self.__mydict[name] self.__dirty_keys.add(name) finally: - self.__dictlock.release() + self._dictlock.release() self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self)) def __iter__(self): try: - self.__dictlock.acquire() + self._dictlock.acquire() return iter(self.__mydict) finally: - self.__dictlock.release() + self._dictlock.release() def __len__(self): try: - self.__dictlock.acquire() + self._dictlock.acquire() return len(self.__mydict) finally: - self.__dictlock.release() + self._dictlock.release() def keys(self): try: - self.__dictlock.acquire() + self._dictlock.acquire() return list(self.__mydict.keys()) finally: - self.__dictlock.release() + self._dictlock.release() def __send_event(self, _event): - for listeners in self.__registered_listeners: + for listeners in self._registered_listeners: listeners(_event.copy()) def __del__(self): self.__send_event(DataPool.Event(DataPool.Event.destroy)) - self.__registered_listeners=None + self._registered_listeners=None def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0): @@ -181,23 +181,17 @@ class DataPool(collections.abc.MutableMapping): else: dump_group = dump_file.create_group(where, name, "DAMARIS data pool") - self.__dictlock.acquire() + self._dictlock.acquire() if incremental: dict_keys=list(self.__dirty_keys) else: # Full sync: current keys + any other dirty keys (e.g. deletions) dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys) + # Optimization: for periodic writes, we only write dirty keys + if write_interval > 0: + dict_keys = list(self.__dirty_keys) - # Apply optimization based on write_interval - if write_interval == 0: - # No periodic writes: always use incremental writing for better performance - dict_keys = list(self.__dirty_keys) - elif write_interval > 0 and not incremental: - # Periodic writes enabled: only write dirty keys that haven't been written yet - # This prevents rewriting objects that were already written in periodic updates - dict_keys = list(self.__dirty_keys) - - self.__dictlock.release() + self._dictlock.release() try: for key in dict_keys: @@ -241,17 +235,17 @@ class DataPool(collections.abc.MutableMapping): # truly new key with collision group_keyname += "_%03d" % extension_count - self.__dictlock.acquire() + self._dictlock.acquire() if key not in self.__mydict: # Key was deleted if key in self.__dirty_keys: self.__dirty_keys.remove(key) - self.__dictlock.release() + self._dictlock.release() continue value=self.__mydict[key] if key in self.__dirty_keys: self.__dirty_keys.remove(key) - self.__dictlock.release() + self._dictlock.release() # now write data, assuming, the object is constant during write operation if hasattr(value, "write_to_hdf"): try: @@ -317,9 +311,9 @@ class DataPool(collections.abc.MutableMapping): self._read_hdf_group(data_pool_group, "") # Clear dirty keys as they are now in sync with HDF5 - self.__dictlock.acquire() + self._dictlock.acquire() self.__dirty_keys.clear() - self.__dictlock.release() + self._dictlock.release() finally: if close_file: @@ -528,8 +522,8 @@ class DataPool(collections.abc.MutableMapping): hdf_file.close() def register_listener(self, listening_function): - self.__registered_listeners.append(listening_function) + self._registered_listeners.append(listening_function) def unregister_listener(self, listening_function): - if listening_function in self.__registered_listeners: - self.__registered_listeners.remove(listening_function) + if listening_function in self._registered_listeners: + self._registered_listeners.remove(listening_function) diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index bf864bf..b81c3a1 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -120,7 +120,7 @@ class LockFile: def del_experiment(self, id): print("Delete experiment..") - self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (self.id,)) + self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (id,)) self.conn.commit() return True @@ -592,39 +592,23 @@ class DamarisGUI: data_cleanup_start = time.time() self.data = None - # Modern listener management - only clean up listeners when truly needed + # Clean up monitor widgets and data pool if hasattr(self, 'monitor') and self.monitor is not None: - # Check if we have significant listener activity - needs_full_cleanup = False listener_count = 0 data_items = 0 - if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None: try: - # Count both listeners and data items for analysis - with self.monitor.data_pool.__dictlock: - listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0 + with self.monitor.data_pool._dictlock: + listener_count = len(self.monitor.data_pool._registered_listeners) if hasattr(self.monitor.data_pool, '_registered_listeners') else 0 data_items = len(self.monitor.data_pool) - needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners - - # Analyze the structure - flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key) - nested_keys = data_items - flat_keys - print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners") except: - needs_full_cleanup = True + pass - if needs_full_cleanup: - print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...") - self.monitor.observe_data_pool( self.data ) - else: - # Efficient cleanup: just reset references without full listener cleanup - print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)") - if hasattr(self.monitor, 'data_pool'): - # Just set to None - listeners will be garbage collected - self.monitor.data_pool = None + # Use observe_data_pool(None) which is now optimized with fast TreeStore replacement + print(f"DEBUG: Data pool cleanup ({listener_count} listeners, {data_items} items)") + self.monitor.observe_data_pool( None ) - print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s") + print(f"DEBUG: Data cleanup took {time.time() - data_cleanup_start:.3f}s") # set the text mark for hdf logging if self.log.textbuffer.get_mark( "lastdumped" ) is None: @@ -2998,10 +2982,17 @@ class MonitorWidgets: self.update_counter_lock = threading.Lock( ) self.description_text = None self.clipping_marker = None + self.source_list_cache = {} def source_list_reset( self ): - self.display_source_treestore.clear( ) - self.source_list_add( 'None' ) + self.display_source_combobox.set_model(None) + try: + # OPTIMIZATION: Replacing the TreeStore is much faster than clear() for large trees + self.display_source_treestore = gtk.TreeStore( gobject.TYPE_STRING ) + self.source_list_cache = {} + self.source_list_add( 'None' ) + finally: + self.display_source_combobox.set_model(self.display_source_treestore) none_iter = self.source_list_find( [ 'None' ] ) if none_iter is not None: self.display_source_combobox.set_active_iter( none_iter ) @@ -3017,17 +3008,56 @@ class MonitorWidgets: def source_list_find( self, namelist ): """ namelist sequence of names, e.g. ["a", "b", "c" ] for "a/b/c" + returns the deepest found iter. namelist is modified (items found are popped). """ + # Try full path from cache first + full_path = "/".join(namelist) + if full_path in self.source_list_cache: + found_iter = self.source_list_cache[full_path] + # Verify it's still valid by checking it belongs to our model + if self.display_source_treestore.iter_is_valid(found_iter): + del namelist[:] + return found_iter + else: + del self.source_list_cache[full_path] + + # Try partial paths from cache + for i in range(len(namelist) - 1, 0, -1): + partial_path = "/".join(namelist[:i]) + if partial_path in self.source_list_cache: + found_iter = self.source_list_cache[partial_path] + if self.display_source_treestore.iter_is_valid(found_iter): + del namelist[:i] + # Continue search from here + retval = found_iter + iter = self.display_source_treestore.iter_children( retval ) + while iter is not None and len( namelist ) > 0: + name = namelist[ 0 ] + iter = self.source_list_find_one( self.display_source_treestore, iter, name ) + if iter is not None: + retval = iter + namelist.pop( 0 ) + # Update cache + partial_path += "/" + name + self.source_list_cache[partial_path] = retval + iter = self.display_source_treestore.iter_children( retval ) + return retval + else: + del self.source_list_cache[partial_path] + model = self.display_source_treestore retval = None #iter = model.get_iter_root( ) iter = model.get_iter_first( ) + current_path = [] while iter is not None and len( namelist ) > 0: name = namelist[ 0 ] iter = self.source_list_find_one( model, iter, name ) if iter is not None: retval = iter namelist.pop( 0 ) + current_path.append(name) + self.source_list_cache["/".join(current_path)] = retval iter = model.iter_children( retval ) return retval @@ -3036,9 +3066,21 @@ class MonitorWidgets: found = self.source_list_find( namelist ) if parent is None: parent = found + + current_path = source_name.split("/") + if found is not None: + # We found part of the path, namelist now contains the rest + # Adjust current_path to reflect the part we already have + already_found_len = len(current_path) - len(namelist) + current_path = current_path[:already_found_len] + else: + current_path = [] + for rest_name in namelist: # append() returns iter for the new row parent = self.display_source_treestore.append( parent, [ rest_name ] ) + current_path.append(rest_name) + self.source_list_cache["/".join(current_path)] = parent def source_list_remove( self, source_name ): namelist = source_name.split( "/" ) @@ -3080,29 +3122,21 @@ class MonitorWidgets: register a listener and save reference to data assume to be in gtk/gdk lock """ - import time - cleanup_start = time.time() gdk.threads_enter( ) try: if self.data_pool is not None: # maybe some extra cleanup needed - print("ToDo: cleanup widgets") - widget_cleanup_start = time.time() if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ): self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener ) self.displayed_data[ 1 ] = None self.displayed_data = [ None, None ] - print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s") - listener_cleanup_start = time.time() self.data_pool.unregister_listener( self.datapool_listener ) self.data_pool = None - print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s") self.update_counter_lock.acquire( ) self.update_counter = 0 self.update_counter_lock.release( ) - print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s") # display states self.source_list_reset( ) @@ -3118,12 +3152,12 @@ class MonitorWidgets: # Populate source list with existing data since the listener # was registered after the data was loaded - for key in sorted(data_pool.keys()): - if key != "__builtins__": # Skip internal keys - self.source_list_add(key) + keys = sorted([k for k in data_pool if k != "__builtins__"]) + for key in keys: + self.source_list_add(key) # Select first data item if available - keys = [k for k in data_pool.keys() if not k.startswith("__")] + keys = [k for k in data_pool if not k.startswith("__")] if keys: keys.sort() first_key = keys[0] @@ -3143,12 +3177,10 @@ class MonitorWidgets: return if debug and self.update_counter < 0: print("negative event count!", self.update_counter) - if self.update_counter > 5: - if debug: - print("sleeping to find time for grapics updates") - threading.Event( ).wait( 0.05 ) - while self.update_counter > 15: - threading.Event( ).wait( 0.05 ) + + # Throttling removed as it blocks data production thread. + # GUI updates are already decoupled via idle_add. + if event.what == DataPool.Event.updated_value: if self.displayed_data[ 0 ] is None or event.subject != self.displayed_data[ 0 ]: # do nothing, forget it