IPKM/python3-damaris
Public Access
3
0
Fork 0
Code Issues 7 Pull Requests Actions Releases Activity

preformance improvement: starting experiment after createing lots of data is now much faster

Browse Source
This commit is contained in:
markusro
2026-07-13 13:56:03 +02:00
parent b04a4274e0
commit 6f7ca7ecaf
2 changed files with 106 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/damaris/gui/DamarisGUI.py
+77 -45
View File
@@ -120,7 +120,7 @@ class LockFile:
def del_experiment(self, id):
print("Delete experiment..")
self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (self.id,))
self.cursor.execute('DELETE FROM damaris WHERE uuid=?', (id,))
self.conn.commit()
return True
@@ -592,39 +592,23 @@ class DamarisGUI:
data_cleanup_start = time.time()
self.data = None
# Modern listener management - only clean up listeners when truly needed
# Clean up monitor widgets and data pool
if hasattr(self, 'monitor') and self.monitor is not None:
# Check if we have significant listener activity
needs_full_cleanup = False
listener_count = 0
data_items = 0
if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None:
try:
# Count both listeners and data items for analysis
with self.monitor.data_pool.__dictlock:
listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0
with self.monitor.data_pool._dictlock:
listener_count = len(self.monitor.data_pool._registered_listeners) if hasattr(self.monitor.data_pool, '_registered_listeners') else 0
data_items = len(self.monitor.data_pool)
needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners
# Analyze the structure
flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key)
nested_keys = data_items - flat_keys
print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners")
except:
needs_full_cleanup = True
pass
if needs_full_cleanup:
print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...")
self.monitor.observe_data_pool( self.data )
else:
# Efficient cleanup: just reset references without full listener cleanup
print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)")
if hasattr(self.monitor, 'data_pool'):
# Just set to None - listeners will be garbage collected
self.monitor.data_pool = None
# Use observe_data_pool(None) which is now optimized with fast TreeStore replacement
print(f"DEBUG: Data pool cleanup ({listener_count} listeners, {data_items} items)")
self.monitor.observe_data_pool( None )
print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s")
print(f"DEBUG: Data cleanup took {time.time() - data_cleanup_start:.3f}s")
# set the text mark for hdf logging
if self.log.textbuffer.get_mark( "lastdumped" ) is None:
@@ -2998,10 +2982,17 @@ class MonitorWidgets:
self.update_counter_lock = threading.Lock( )
self.description_text = None
self.clipping_marker = None
self.source_list_cache = {}
def source_list_reset( self ):
self.display_source_treestore.clear( )
self.source_list_add( 'None' )
self.display_source_combobox.set_model(None)
try:
# OPTIMIZATION: Replacing the TreeStore is much faster than clear() for large trees
self.display_source_treestore = gtk.TreeStore( gobject.TYPE_STRING )
self.source_list_cache = {}
self.source_list_add( 'None' )
finally:
self.display_source_combobox.set_model(self.display_source_treestore)
none_iter = self.source_list_find( [ 'None' ] )
if none_iter is not None:
self.display_source_combobox.set_active_iter( none_iter )
@@ -3017,17 +3008,56 @@ class MonitorWidgets:
def source_list_find( self, namelist ):
"""
namelist sequence of names, e.g. ["a", "b", "c" ] for "a/b/c"
returns the deepest found iter. namelist is modified (items found are popped).
"""
# Try full path from cache first
full_path = "/".join(namelist)
if full_path in self.source_list_cache:
found_iter = self.source_list_cache[full_path]
# Verify it's still valid by checking it belongs to our model
if self.display_source_treestore.iter_is_valid(found_iter):
del namelist[:]
return found_iter
else:
del self.source_list_cache[full_path]
# Try partial paths from cache
for i in range(len(namelist) - 1, 0, -1):
partial_path = "/".join(namelist[:i])
if partial_path in self.source_list_cache:
found_iter = self.source_list_cache[partial_path]
if self.display_source_treestore.iter_is_valid(found_iter):
del namelist[:i]
# Continue search from here
retval = found_iter
iter = self.display_source_treestore.iter_children( retval )
while iter is not None and len( namelist ) > 0:
name = namelist[ 0 ]
iter = self.source_list_find_one( self.display_source_treestore, iter, name )
if iter is not None:
retval = iter
namelist.pop( 0 )
# Update cache
partial_path += "/" + name
self.source_list_cache[partial_path] = retval
iter = self.display_source_treestore.iter_children( retval )
return retval
else:
del self.source_list_cache[partial_path]
model = self.display_source_treestore
retval = None
#iter = model.get_iter_root( )
iter = model.get_iter_first( )
current_path = []
while iter is not None and len( namelist ) > 0:
name = namelist[ 0 ]
iter = self.source_list_find_one( model, iter, name )
if iter is not None:
retval = iter
namelist.pop( 0 )
current_path.append(name)
self.source_list_cache["/".join(current_path)] = retval
iter = model.iter_children( retval )
return retval
@@ -3036,9 +3066,21 @@ class MonitorWidgets:
found = self.source_list_find( namelist )
if parent is None:
parent = found
current_path = source_name.split("/")
if found is not None:
# We found part of the path, namelist now contains the rest
# Adjust current_path to reflect the part we already have
already_found_len = len(current_path) - len(namelist)
current_path = current_path[:already_found_len]
else:
current_path = []
for rest_name in namelist:
# append() returns iter for the new row
parent = self.display_source_treestore.append( parent, [ rest_name ] )
current_path.append(rest_name)
self.source_list_cache["/".join(current_path)] = parent
def source_list_remove( self, source_name ):
namelist = source_name.split( "/" )
@@ -3080,29 +3122,21 @@ class MonitorWidgets:
register a listener and save reference to data
assume to be in gtk/gdk lock
"""
import time
cleanup_start = time.time()
gdk.threads_enter( )
try:
if self.data_pool is not None:
# maybe some extra cleanup needed
print("ToDo: cleanup widgets")
widget_cleanup_start = time.time()
if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
self.displayed_data[ 1 ] = None
self.displayed_data = [ None, None ]
print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s")
listener_cleanup_start = time.time()
self.data_pool.unregister_listener( self.datapool_listener )
self.data_pool = None
print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s")
self.update_counter_lock.acquire( )
self.update_counter = 0
self.update_counter_lock.release( )
print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s")
# display states
self.source_list_reset( )
@@ -3118,12 +3152,12 @@ class MonitorWidgets:
# Populate source list with existing data since the listener
# was registered after the data was loaded
for key in sorted(data_pool.keys()):
if key != "__builtins__": # Skip internal keys
self.source_list_add(key)
keys = sorted([k for k in data_pool if k != "__builtins__"])
for key in keys:
self.source_list_add(key)
# Select first data item if available
keys = [k for k in data_pool.keys() if not k.startswith("__")]
keys = [k for k in data_pool if not k.startswith("__")]
if keys:
keys.sort()
first_key = keys[0]
@@ -3143,12 +3177,10 @@ class MonitorWidgets:
return
if debug and self.update_counter < 0:
print("negative event count!", self.update_counter)
if self.update_counter > 5:
if debug:
print("sleeping to find time for grapics updates")
threading.Event( ).wait( 0.05 )
while self.update_counter > 15:
threading.Event( ).wait( 0.05 )
# Throttling removed as it blocks data production thread.
# GUI updates are already decoupled via idle_add.
if event.what == DataPool.Event.updated_value:
if self.displayed_data[ 0 ] is None or event.subject != self.displayed_data[ 0 ]:
# do nothing, forget it