IPKM/python3-damaris
Public Access
3
0
Fork 0
Code Issues 7 Pull Requests Actions Releases Activity

Merge performance

Browse Source 
Merge branch 'main' of gitea.pkm.physik.tu-darmstadt.de:IPKM/python3-damaris
This commit is contained in:
markusro
2026-07-13 13:12:51 +02:00
parent 02409488b8 ebc2daf170
commit b04a4274e0
2 changed files with 108 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/damaris/data/DataPool.py
+22 -1
View File
@@ -110,7 +110,7 @@ class DataPool(collections.abc.MutableMapping):
self.__send_event(DataPool.Event(DataPool.Event.destroy))
self.__registered_listeners=None
def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False):
def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0):
"""
Writes the content of a DataPool object to an HDF5 file using the `tables` library.
@@ -143,6 +143,17 @@ class DataPool(collections.abc.MutableMapping):
Compression level to use when applying compression. Has no effect if `complib`
is set to None.
incremental : bool, optional
If True, only write keys that have been modified since the last write (dirty keys).
If False, write all keys. Defaults to False.
write_interval : int, optional
Time interval in seconds between periodic writes. If 0, no periodic writes are performed.
This parameter affects the writing strategy:
- If write_interval is 0: Always use incremental writing for better performance
- If write_interval > 0: Only write dirty keys that haven't been written yet
Defaults to 0.
Raises:
Exception
If `hdffile` is neither a string nor a valid `tables.File` object.
@@ -176,6 +187,16 @@ class DataPool(collections.abc.MutableMapping):
else:
# Full sync: current keys + any other dirty keys (e.g. deletions)
dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
# Apply optimization based on write_interval
if write_interval == 0:
# No periodic writes: always use incremental writing for better performance
dict_keys = list(self.__dirty_keys)
elif write_interval > 0 and not incremental:
# Periodic writes enabled: only write dirty keys that haven't been written yet
# This prevents rewriting objects that were already written in periodic updates
dict_keys = list(self.__dirty_keys)
self.__dictlock.release()
try:
src/damaris/gui/DamarisGUI.py
+86 -4
View File
@@ -501,6 +501,9 @@ class DamarisGUI:
dialog.destroy()
def start_experiment( self, widget, data=None ):
import time
overall_start = time.time()
print(f"DEBUG: start_experiment called at {overall_start}")
# something running?
if self.si is not None:
@@ -508,11 +511,15 @@ class DamarisGUI:
self.si = None
# get config values:
config_start = time.time()
actual_config = self.config.get( )
print(f"DEBUG: Config load took {time.time() - config_start:.3f}s")
# get scripts and start script interface
gui_start = time.time()
self.sw.disable_editing( )
exp_script, res_script = self.sw.get_scripts( )
print(f"DEBUG: GUI operations took {time.time() - gui_start:.3f}s")
if not actual_config[ "start_result_script" ]:
res_script = ""
if not actual_config[ "start_experiment_script" ]:
@@ -528,6 +535,7 @@ class DamarisGUI:
# check whether scripts are syntactically valid
# should be merged with check script function
compile_start = time.time()
exp_code = None
if exp_script != "":
try:
@@ -547,6 +555,7 @@ class DamarisGUI:
res_code = None
if res_script != "":
# Continue with result script compilation
try:
res_code = compile( res_script, "Result Script", "exec" )
except SyntaxError as e:
@@ -563,6 +572,8 @@ class DamarisGUI:
pass
print(e)
print(f"DEBUG: Script compilation took {time.time() - compile_start:.3f}s")
# detect error
if (exp_script != "" and exp_code is None) or \
(res_script != "" and res_code is None):
@@ -577,9 +588,43 @@ class DamarisGUI:
self.toolbar_pause_button.set_sensitive( True )
self.toolbar_pause_button.set_active( False )
# delete old data
# delete old data - use efficient cleanup
data_cleanup_start = time.time()
self.data = None
self.monitor.observe_data_pool( self.data )
# Modern listener management - only clean up listeners when truly needed
if hasattr(self, 'monitor') and self.monitor is not None:
# Check if we have significant listener activity
needs_full_cleanup = False
listener_count = 0
data_items = 0
if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None:
try:
# Count both listeners and data items for analysis
with self.monitor.data_pool.__dictlock:
listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0
data_items = len(self.monitor.data_pool)
needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners
# Analyze the structure
flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key)
nested_keys = data_items - flat_keys
print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners")
except:
needs_full_cleanup = True
if needs_full_cleanup:
print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...")
self.monitor.observe_data_pool( self.data )
else:
# Efficient cleanup: just reset references without full listener cleanup
print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)")
if hasattr(self.monitor, 'data_pool'):
# Just set to None - listeners will be garbage collected
self.monitor.data_pool = None
print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s")
# set the text mark for hdf logging
if self.log.textbuffer.get_mark( "lastdumped" ) is None:
@@ -590,7 +635,16 @@ class DamarisGUI:
interval = 0.1
self.stop_experiment_flag.clear()
# Add timing debug info
import time
start_time = time.time()
print(f"DEBUG: Starting experiment at {start_time}")
self.id = self.lockfile.add_experiment()
print(f"DEBUG: Lock file operation took {time.time() - start_time:.3f}s")
# Add more timing points
lock_wait_start = time.time()
while not self.lockfile.am_i_next():
while gtk.events_pending():
gtk.main_iteration_do(False)
@@ -601,7 +655,9 @@ class DamarisGUI:
loop_run = 0
loop_run += interval
if self.stop_experiment_flag.isSet():
print(f"DEBUG: Lock wait took {time.time() - lock_wait_start:.3f}s")
if self.stop_experiment_flag.isSet():
#Experiment has been stopped, clean up and leave
self.experiment_script_statusbar_label.set_text("Experiment stopped")
self.state = DamarisGUI.Edit_State
@@ -618,19 +674,26 @@ class DamarisGUI:
try:
self.spool_dir = os.path.abspath( actual_config[ "spool_dir" ] )
# setup script engines
script_interface_start = time.time()
self.si = ScriptInterface( exp_code,
res_code,
backend,
self.spool_dir,
clear_jobs=actual_config[ "del_jobs_after_execution" ],
clear_results=actual_config[ "del_results_after_processing" ] )
print(f"DEBUG: ScriptInterface creation took {time.time() - script_interface_start:.3f}s")
self.data = self.si.data
# run frontend and script engines
self.monitor.observe_data_pool( self.data )
# Start temperature monitoring if configured (before scripts start)
temp_start = time.time()
self.start_temperature_monitoring(actual_config)
print(f"DEBUG: Temperature monitoring startup took {time.time() - temp_start:.3f}s")
scripts_start = time.time()
self.si.runScripts( )
print(f"DEBUG: Script execution startup took {time.time() - scripts_start:.3f}s")
except Exception as e:
#print "ToDo evaluate exception",str(e), "at",traceback.extract_tb(sys.exc_info()[2])[-1][1:3]
#print "Full traceback:"
@@ -807,6 +870,11 @@ class DamarisGUI:
self.stop_temperature_monitoring()
if self.save_thread is None and self.dump_filename != "":
print("all subprocesses ended, saving data pool")
# Disable run button during data saving to prevent UI confusion
gdk.threads_enter()
self.toolbar_run_button.set_sensitive(False)
gdk.threads_leave()
# thread to save data...
self.save_thread = threading.Thread( target=self.dump_states, name="dump states" )
self.save_thread.start( )
@@ -1012,7 +1080,13 @@ class DamarisGUI:
# save the data!
self.data.write_hdf5( dump_file, where="/", name="data_pool",
complib=self.dump_complib, complevel=self.dump_complevel,
incremental=not init )
incremental=not init, write_interval=self.dump_timeinterval )
# Re-enable run button if this was called from save thread
if not init and threading.current_thread().name == "dump states":
gdk.threads_enter()
self.toolbar_run_button.set_sensitive(True)
gdk.threads_leave()
# now save additional information
e = self.si.data.get( "__recentexperiment", -1 ) + 1
@@ -3006,21 +3080,29 @@ class MonitorWidgets:
register a listener and save reference to data
assume to be in gtk/gdk lock
"""
import time
cleanup_start = time.time()
gdk.threads_enter( )
try:
if self.data_pool is not None:
# maybe some extra cleanup needed
print("ToDo: cleanup widgets")
widget_cleanup_start = time.time()
if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
self.displayed_data[ 1 ] = None
self.displayed_data = [ None, None ]
print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s")
listener_cleanup_start = time.time()
self.data_pool.unregister_listener( self.datapool_listener )
self.data_pool = None
print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s")
self.update_counter_lock.acquire( )
self.update_counter = 0
self.update_counter_lock.release( )
print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s")
# display states
self.source_list_reset( )