IPKM/python3-damaris
Public Access
3
0
Fork 0
Code Issues 7 Pull Requests Actions Releases Activity

preformance improvement: starting experiment after createing lots of data is now much faster

Browse Source
This commit is contained in:
markusro
2026-07-13 13:56:03 +02:00
parent b04a4274e0
commit 6f7ca7ecaf
2 changed files with 106 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/damaris/data/DataPool.py
+29 -35
View File
@@ -47,20 +47,20 @@ class DataPool(collections.abc.MutableMapping):
def __init__(self): def __init__(self):
self.__mydict={} self.__mydict={}
self.__dictlock=threading.Lock() self._dictlock=threading.RLock()
self.__registered_listeners=[] self._registered_listeners=[]
self.__dirty_keys=set() self.__dirty_keys=set()
def __getitem__(self, name): def __getitem__(self, name):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
return self.__mydict[name] return self.__mydict[name]
finally: finally:
self.__dictlock.release() self._dictlock.release()
def __setitem__(self, name, value): def __setitem__(self, name, value):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
if name in self.__mydict: if name in self.__mydict:
e=DataPool.Event(DataPool.Event.updated_value,name,self) e=DataPool.Event(DataPool.Event.updated_value,name,self)
else: else:
@@ -68,47 +68,47 @@ class DataPool(collections.abc.MutableMapping):
self.__mydict[name]=value self.__mydict[name]=value
self.__dirty_keys.add(name) self.__dirty_keys.add(name)
finally: finally:
self.__dictlock.release() self._dictlock.release()
self.__send_event(e) self.__send_event(e)
def __delitem__(self, name): def __delitem__(self, name):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
del self.__mydict[name] del self.__mydict[name]
self.__dirty_keys.add(name) self.__dirty_keys.add(name)
finally: finally:
self.__dictlock.release() self._dictlock.release()
self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self)) self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self))
def __iter__(self): def __iter__(self):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
return iter(self.__mydict) return iter(self.__mydict)
finally: finally:
self.__dictlock.release() self._dictlock.release()
def __len__(self): def __len__(self):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
return len(self.__mydict) return len(self.__mydict)
finally: finally:
self.__dictlock.release() self._dictlock.release()
def keys(self): def keys(self):
try: try:
self.__dictlock.acquire() self._dictlock.acquire()
return list(self.__mydict.keys()) return list(self.__mydict.keys())
finally: finally:
self.__dictlock.release() self._dictlock.release()
def __send_event(self, _event): def __send_event(self, _event):
for listeners in self.__registered_listeners: for listeners in self._registered_listeners:
listeners(_event.copy()) listeners(_event.copy())
def __del__(self): def __del__(self):
self.__send_event(DataPool.Event(DataPool.Event.destroy)) self.__send_event(DataPool.Event(DataPool.Event.destroy))
self.__registered_listeners=None self._registered_listeners=None
def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0): def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0):
@@ -181,23 +181,17 @@ class DataPool(collections.abc.MutableMapping):
else: else:
dump_group = dump_file.create_group(where, name, "DAMARIS data pool") dump_group = dump_file.create_group(where, name, "DAMARIS data pool")
self.__dictlock.acquire() self._dictlock.acquire()
if incremental: if incremental:
dict_keys=list(self.__dirty_keys) dict_keys=list(self.__dirty_keys)
else: else:
# Full sync: current keys + any other dirty keys (e.g. deletions) # Full sync: current keys + any other dirty keys (e.g. deletions)
dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys) dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
# Optimization: for periodic writes, we only write dirty keys
if write_interval > 0:
dict_keys = list(self.__dirty_keys)
# Apply optimization based on write_interval self._dictlock.release()
if write_interval == 0:
# No periodic writes: always use incremental writing for better performance
dict_keys = list(self.__dirty_keys)
elif write_interval > 0 and not incremental:
# Periodic writes enabled: only write dirty keys that haven't been written yet
# This prevents rewriting objects that were already written in periodic updates
dict_keys = list(self.__dirty_keys)
self.__dictlock.release()
try: try:
for key in dict_keys: for key in dict_keys:
@@ -241,17 +235,17 @@ class DataPool(collections.abc.MutableMapping):
# truly new key with collision # truly new key with collision
group_keyname += "_%03d" % extension_count group_keyname += "_%03d" % extension_count
self.__dictlock.acquire() self._dictlock.acquire()
if key not in self.__mydict: if key not in self.__mydict:
# Key was deleted # Key was deleted
if key in self.__dirty_keys: if key in self.__dirty_keys:
self.__dirty_keys.remove(key) self.__dirty_keys.remove(key)
self.__dictlock.release() self._dictlock.release()
continue continue
value=self.__mydict[key] value=self.__mydict[key]
if key in self.__dirty_keys: if key in self.__dirty_keys:
self.__dirty_keys.remove(key) self.__dirty_keys.remove(key)
self.__dictlock.release() self._dictlock.release()
# now write data, assuming, the object is constant during write operation # now write data, assuming, the object is constant during write operation
if hasattr(value, "write_to_hdf"): if hasattr(value, "write_to_hdf"):
try: try:
@@ -317,9 +311,9 @@ class DataPool(collections.abc.MutableMapping):
self._read_hdf_group(data_pool_group, "") self._read_hdf_group(data_pool_group, "")
# Clear dirty keys as they are now in sync with HDF5 # Clear dirty keys as they are now in sync with HDF5
self.__dictlock.acquire() self._dictlock.acquire()
self.__dirty_keys.clear() self.__dirty_keys.clear()
self.__dictlock.release() self._dictlock.release()
finally: finally:
if close_file: if close_file:
@@ -528,8 +522,8 @@ class DataPool(collections.abc.MutableMapping):
hdf_file.close() hdf_file.close()
def register_listener(self, listening_function): def register_listener(self, listening_function):
self.__registered_listeners.append(listening_function) self._registered_listeners.append(listening_function)
def unregister_listener(self, listening_function): def unregister_listener(self, listening_function):
if listening_function in self.__registered_listeners: if listening_function in self._registered_listeners:
self.__registered_listeners.remove(listening_function) self._registered_listeners.remove(listening_function)
src/damaris/gui/DamarisGUI.py
+77 -45
View File
@@ -120,7 +120,7 @@ class LockFile:
def del_experiment(self, id): def del_experiment(self, id):
print("Delete experiment..") print("Delete experiment..")
self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (self.id,)) self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (id,))
self.conn.commit() self.conn.commit()
return True return True
@@ -592,39 +592,23 @@ class DamarisGUI:
data_cleanup_start = time.time() data_cleanup_start = time.time()
self.data = None self.data = None
# Modern listener management - only clean up listeners when truly needed # Clean up monitor widgets and data pool
if hasattr(self, 'monitor') and self.monitor is not None: if hasattr(self, 'monitor') and self.monitor is not None:
# Check if we have significant listener activity
needs_full_cleanup = False
listener_count = 0 listener_count = 0
data_items = 0 data_items = 0
if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None: if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None:
try: try:
# Count both listeners and data items for analysis with self.monitor.data_pool._dictlock:
with self.monitor.data_pool.__dictlock: listener_count = len(self.monitor.data_pool._registered_listeners) if hasattr(self.monitor.data_pool, '_registered_listeners') else 0
listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0
data_items = len(self.monitor.data_pool) data_items = len(self.monitor.data_pool)
needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners
# Analyze the structure
flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key)
nested_keys = data_items - flat_keys
print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners")
except: except:
needs_full_cleanup = True pass
if needs_full_cleanup: # Use observe_data_pool(None) which is now optimized with fast TreeStore replacement
print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...") print(f"DEBUG: Data pool cleanup ({listener_count} listeners, {data_items} items)")
self.monitor.observe_data_pool( self.data ) self.monitor.observe_data_pool( None )
else:
# Efficient cleanup: just reset references without full listener cleanup
print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)")
if hasattr(self.monitor, 'data_pool'):
# Just set to None - listeners will be garbage collected
self.monitor.data_pool = None
print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s") print(f"DEBUG: Data cleanup took {time.time() - data_cleanup_start:.3f}s")
# set the text mark for hdf logging # set the text mark for hdf logging
if self.log.textbuffer.get_mark( "lastdumped" ) is None: if self.log.textbuffer.get_mark( "lastdumped" ) is None:
@@ -2998,10 +2982,17 @@ class MonitorWidgets:
self.update_counter_lock = threading.Lock( ) self.update_counter_lock = threading.Lock( )
self.description_text = None self.description_text = None
self.clipping_marker = None self.clipping_marker = None
self.source_list_cache = {}
def source_list_reset( self ): def source_list_reset( self ):
self.display_source_treestore.clear( ) self.display_source_combobox.set_model(None)
self.source_list_add( 'None' ) try:
# OPTIMIZATION: Replacing the TreeStore is much faster than clear() for large trees
self.display_source_treestore = gtk.TreeStore( gobject.TYPE_STRING )
self.source_list_cache = {}
self.source_list_add( 'None' )
finally:
self.display_source_combobox.set_model(self.display_source_treestore)
none_iter = self.source_list_find( [ 'None' ] ) none_iter = self.source_list_find( [ 'None' ] )
if none_iter is not None: if none_iter is not None:
self.display_source_combobox.set_active_iter( none_iter ) self.display_source_combobox.set_active_iter( none_iter )
@@ -3017,17 +3008,56 @@ class MonitorWidgets:
def source_list_find( self, namelist ): def source_list_find( self, namelist ):
""" """
namelist sequence of names, e.g. ["a", "b", "c" ] for "a/b/c" namelist sequence of names, e.g. ["a", "b", "c" ] for "a/b/c"
returns the deepest found iter. namelist is modified (items found are popped).
""" """
# Try full path from cache first
full_path = "/".join(namelist)
if full_path in self.source_list_cache:
found_iter = self.source_list_cache[full_path]
# Verify it's still valid by checking it belongs to our model
if self.display_source_treestore.iter_is_valid(found_iter):
del namelist[:]
return found_iter
else:
del self.source_list_cache[full_path]
# Try partial paths from cache
for i in range(len(namelist) - 1, 0, -1):
partial_path = "/".join(namelist[:i])
if partial_path in self.source_list_cache:
found_iter = self.source_list_cache[partial_path]
if self.display_source_treestore.iter_is_valid(found_iter):
del namelist[:i]
# Continue search from here
retval = found_iter
iter = self.display_source_treestore.iter_children( retval )
while iter is not None and len( namelist ) > 0:
name = namelist[ 0 ]
iter = self.source_list_find_one( self.display_source_treestore, iter, name )
if iter is not None:
retval = iter
namelist.pop( 0 )
# Update cache
partial_path += "/" + name
self.source_list_cache[partial_path] = retval
iter = self.display_source_treestore.iter_children( retval )
return retval
else:
del self.source_list_cache[partial_path]
model = self.display_source_treestore model = self.display_source_treestore
retval = None retval = None
#iter = model.get_iter_root( ) #iter = model.get_iter_root( )
iter = model.get_iter_first( ) iter = model.get_iter_first( )
current_path = []
while iter is not None and len( namelist ) > 0: while iter is not None and len( namelist ) > 0:
name = namelist[ 0 ] name = namelist[ 0 ]
iter = self.source_list_find_one( model, iter, name ) iter = self.source_list_find_one( model, iter, name )
if iter is not None: if iter is not None:
retval = iter retval = iter
namelist.pop( 0 ) namelist.pop( 0 )
current_path.append(name)
self.source_list_cache["/".join(current_path)] = retval
iter = model.iter_children( retval ) iter = model.iter_children( retval )
return retval return retval
@@ -3036,9 +3066,21 @@ class MonitorWidgets:
found = self.source_list_find( namelist ) found = self.source_list_find( namelist )
if parent is None: if parent is None:
parent = found parent = found
current_path = source_name.split("/")
if found is not None:
# We found part of the path, namelist now contains the rest
# Adjust current_path to reflect the part we already have
already_found_len = len(current_path) - len(namelist)
current_path = current_path[:already_found_len]
else:
current_path = []
for rest_name in namelist: for rest_name in namelist:
# append() returns iter for the new row # append() returns iter for the new row
parent = self.display_source_treestore.append( parent, [ rest_name ] ) parent = self.display_source_treestore.append( parent, [ rest_name ] )
current_path.append(rest_name)
self.source_list_cache["/".join(current_path)] = parent
def source_list_remove( self, source_name ): def source_list_remove( self, source_name ):
namelist = source_name.split( "/" ) namelist = source_name.split( "/" )
@@ -3080,29 +3122,21 @@ class MonitorWidgets:
register a listener and save reference to data register a listener and save reference to data
assume to be in gtk/gdk lock assume to be in gtk/gdk lock
""" """
import time
cleanup_start = time.time()
gdk.threads_enter( ) gdk.threads_enter( )
try: try:
if self.data_pool is not None: if self.data_pool is not None:
# maybe some extra cleanup needed # maybe some extra cleanup needed
print("ToDo: cleanup widgets")
widget_cleanup_start = time.time()
if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ): if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener ) self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
self.displayed_data[ 1 ] = None self.displayed_data[ 1 ] = None
self.displayed_data = [ None, None ] self.displayed_data = [ None, None ]
print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s")
listener_cleanup_start = time.time()
self.data_pool.unregister_listener( self.datapool_listener ) self.data_pool.unregister_listener( self.datapool_listener )
self.data_pool = None self.data_pool = None
print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s")
self.update_counter_lock.acquire( ) self.update_counter_lock.acquire( )
self.update_counter = 0 self.update_counter = 0
self.update_counter_lock.release( ) self.update_counter_lock.release( )
print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s")
# display states # display states
self.source_list_reset( ) self.source_list_reset( )
@@ -3118,12 +3152,12 @@ class MonitorWidgets:
# Populate source list with existing data since the listener # Populate source list with existing data since the listener
# was registered after the data was loaded # was registered after the data was loaded
for key in sorted(data_pool.keys()): keys = sorted([k for k in data_pool if k != "__builtins__"])
if key != "__builtins__": # Skip internal keys for key in keys:
self.source_list_add(key) self.source_list_add(key)
# Select first data item if available # Select first data item if available
keys = [k for k in data_pool.keys() if not k.startswith("__")] keys = [k for k in data_pool if not k.startswith("__")]
if keys: if keys:
keys.sort() keys.sort()
first_key = keys[0] first_key = keys[0]
@@ -3143,12 +3177,10 @@ class MonitorWidgets:
return return
if debug and self.update_counter < 0: if debug and self.update_counter < 0:
print("negative event count!", self.update_counter) print("negative event count!", self.update_counter)
if self.update_counter > 5:
if debug: # Throttling removed as it blocks data production thread.
print("sleeping to find time for grapics updates") # GUI updates are already decoupled via idle_add.
threading.Event( ).wait( 0.05 )
while self.update_counter > 15:
threading.Event( ).wait( 0.05 )
if event.what == DataPool.Event.updated_value: if event.what == DataPool.Event.updated_value:
if self.displayed_data[ 0 ] is None or event.subject != self.displayed_data[ 0 ]: if self.displayed_data[ 0 ] is None or event.subject != self.displayed_data[ 0 ]:
# do nothing, forget it # do nothing, forget it