From a1323a7a5da5f7d859f1a369801e2437e4f1c3a3 Mon Sep 17 00:00:00 2001 From: Markus Rosenstihl Date: Sun, 12 Jul 2026 21:04:48 +0200 Subject: [PATCH 1/4] debug timing added for performance test --- src/damaris/data/DataPool.py | 23 +++++++++++++++++- src/damaris/gui/DamarisGUI.py | 44 +++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/damaris/data/DataPool.py b/src/damaris/data/DataPool.py index c454dd4..6a0c73a 100644 --- a/src/damaris/data/DataPool.py +++ b/src/damaris/data/DataPool.py @@ -110,7 +110,7 @@ class DataPool(collections.abc.MutableMapping): self.__send_event(DataPool.Event(DataPool.Event.destroy)) self.__registered_listeners=None - def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False): + def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0): """ Writes the content of a DataPool object to an HDF5 file using the `tables` library. @@ -143,6 +143,17 @@ class DataPool(collections.abc.MutableMapping): Compression level to use when applying compression. Has no effect if `complib` is set to None. + incremental : bool, optional + If True, only write keys that have been modified since the last write (dirty keys). + If False, write all keys. Defaults to False. + + write_interval : int, optional + Time interval in seconds between periodic writes. If 0, no periodic writes are performed. + This parameter affects the writing strategy: + - If write_interval is 0: Always use incremental writing for better performance + - If write_interval > 0: Only write dirty keys that haven't been written yet + Defaults to 0. + Raises: Exception If `hdffile` is neither a string nor a valid `tables.File` object. @@ -176,6 +187,16 @@ class DataPool(collections.abc.MutableMapping): else: # Full sync: current keys + any other dirty keys (e.g. deletions) dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys) + + # Apply optimization based on write_interval + if write_interval == 0: + # No periodic writes: always use incremental writing for better performance + dict_keys = list(self.__dirty_keys) + elif write_interval > 0 and not incremental: + # Periodic writes enabled: only write dirty keys that haven't been written yet + # This prevents rewriting objects that were already written in periodic updates + dict_keys = list(self.__dirty_keys) + self.__dictlock.release() try: diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index 82f486c..7fa2a70 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -501,6 +501,9 @@ class DamarisGUI: dialog.destroy() def start_experiment( self, widget, data=None ): + import time + overall_start = time.time() + print(f"DEBUG: start_experiment called at {overall_start}") # something running? if self.si is not None: @@ -508,11 +511,15 @@ class DamarisGUI: self.si = None # get config values: + config_start = time.time() actual_config = self.config.get( ) + print(f"DEBUG: Config load took {time.time() - config_start:.3f}s") # get scripts and start script interface + gui_start = time.time() self.sw.disable_editing( ) exp_script, res_script = self.sw.get_scripts( ) + print(f"DEBUG: GUI operations took {time.time() - gui_start:.3f}s") if not actual_config[ "start_result_script" ]: res_script = "" if not actual_config[ "start_experiment_script" ]: @@ -528,6 +535,7 @@ class DamarisGUI: # check whether scripts are syntactically valid # should be merged with check script function + compile_start = time.time() exp_code = None if exp_script != "": try: @@ -547,6 +555,7 @@ class DamarisGUI: res_code = None if res_script != "": + # Continue with result script compilation try: res_code = compile( res_script, "Result Script", "exec" ) except SyntaxError as e: @@ -563,6 +572,8 @@ class DamarisGUI: pass print(e) + print(f"DEBUG: Script compilation took {time.time() - compile_start:.3f}s") + # detect error if (exp_script != "" and exp_code is None) or \ (res_script != "" and res_code is None): @@ -590,7 +601,16 @@ class DamarisGUI: interval = 0.1 self.stop_experiment_flag.clear() + # Add timing debug info + import time + start_time = time.time() + print(f"DEBUG: Starting experiment at {start_time}") + self.id = self.lockfile.add_experiment() + print(f"DEBUG: Lock file operation took {time.time() - start_time:.3f}s") + + # Add more timing points + lock_wait_start = time.time() while not self.lockfile.am_i_next(): while gtk.events_pending(): gtk.main_iteration_do(False) @@ -601,7 +621,9 @@ class DamarisGUI: loop_run = 0 loop_run += interval - if self.stop_experiment_flag.isSet(): + print(f"DEBUG: Lock wait took {time.time() - lock_wait_start:.3f}s") + + if self.stop_experiment_flag.isSet(): #Experiment has been stopped, clean up and leave self.experiment_script_statusbar_label.set_text("Experiment stopped") self.state = DamarisGUI.Edit_State @@ -618,19 +640,26 @@ class DamarisGUI: try: self.spool_dir = os.path.abspath( actual_config[ "spool_dir" ] ) # setup script engines + script_interface_start = time.time() self.si = ScriptInterface( exp_code, res_code, backend, self.spool_dir, clear_jobs=actual_config[ "del_jobs_after_execution" ], clear_results=actual_config[ "del_results_after_processing" ] ) + print(f"DEBUG: ScriptInterface creation took {time.time() - script_interface_start:.3f}s") self.data = self.si.data # run frontend and script engines self.monitor.observe_data_pool( self.data ) # Start temperature monitoring if configured (before scripts start) + temp_start = time.time() self.start_temperature_monitoring(actual_config) + print(f"DEBUG: Temperature monitoring startup took {time.time() - temp_start:.3f}s") + + scripts_start = time.time() self.si.runScripts( ) + print(f"DEBUG: Script execution startup took {time.time() - scripts_start:.3f}s") except Exception as e: #print "ToDo evaluate exception",str(e), "at",traceback.extract_tb(sys.exc_info()[2])[-1][1:3] #print "Full traceback:" @@ -807,6 +836,11 @@ class DamarisGUI: self.stop_temperature_monitoring() if self.save_thread is None and self.dump_filename != "": print("all subprocesses ended, saving data pool") + # Disable run button during data saving to prevent UI confusion + gdk.threads_enter() + self.toolbar_run_button.set_sensitive(False) + gdk.threads_leave() + # thread to save data... self.save_thread = threading.Thread( target=self.dump_states, name="dump states" ) self.save_thread.start( ) @@ -1012,7 +1046,13 @@ class DamarisGUI: # save the data! self.data.write_hdf5( dump_file, where="/", name="data_pool", complib=self.dump_complib, complevel=self.dump_complevel, - incremental=not init ) + incremental=not init, write_interval=self.dump_timeinterval ) + + # Re-enable run button if this was called from save thread + if not init and threading.current_thread().name == "dump states": + gdk.threads_enter() + self.toolbar_run_button.set_sensitive(True) + gdk.threads_leave() # now save additional information e = self.si.data.get( "__recentexperiment", -1 ) + 1 From 3d847588c47457327c6e72b0a31477859f29ab00 Mon Sep 17 00:00:00 2001 From: Markus Rosenstihl Date: Sun, 12 Jul 2026 21:07:41 +0200 Subject: [PATCH 2/4] debug timing added for performance test --- src/damaris/gui/DamarisGUI.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index 7fa2a70..7a9fbb1 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -589,8 +589,10 @@ class DamarisGUI: self.toolbar_pause_button.set_active( False ) # delete old data + data_cleanup_start = time.time() self.data = None self.monitor.observe_data_pool( self.data ) + print(f"DEBUG: Data cleanup took {time.time() - data_cleanup_start:.3f}s") # set the text mark for hdf logging if self.log.textbuffer.get_mark( "lastdumped" ) is None: @@ -3046,21 +3048,29 @@ class MonitorWidgets: register a listener and save reference to data assume to be in gtk/gdk lock """ + import time + cleanup_start = time.time() gdk.threads_enter( ) try: if self.data_pool is not None: # maybe some extra cleanup needed print("ToDo: cleanup widgets") + widget_cleanup_start = time.time() if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ): self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener ) self.displayed_data[ 1 ] = None self.displayed_data = [ None, None ] + print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s") + + listener_cleanup_start = time.time() self.data_pool.unregister_listener( self.datapool_listener ) self.data_pool = None + print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s") self.update_counter_lock.acquire( ) self.update_counter = 0 self.update_counter_lock.release( ) + print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s") # display states self.source_list_reset( ) From a75917efd5f9437da8b94c20d0450c114ecb1be7 Mon Sep 17 00:00:00 2001 From: Markus Rosenstihl Date: Sun, 12 Jul 2026 22:40:32 +0200 Subject: [PATCH 3/4] preformance improvment when using many data_pool objects --- src/damaris/gui/DamarisGUI.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index 7a9fbb1..a206efb 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -588,11 +588,36 @@ class DamarisGUI: self.toolbar_pause_button.set_sensitive( True ) self.toolbar_pause_button.set_active( False ) - # delete old data + # delete old data - use efficient cleanup data_cleanup_start = time.time() self.data = None - self.monitor.observe_data_pool( self.data ) - print(f"DEBUG: Data cleanup took {time.time() - data_cleanup_start:.3f}s") + + # Modern listener management - only clean up listeners when truly needed + if hasattr(self, 'monitor') and self.monitor is not None: + # Check if we have significant listener activity + needs_full_cleanup = False + listener_count = 0 + + if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None: + try: + # Count actual listeners instead of just items + with self.monitor.data_pool.__dictlock: + listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0 + needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners + except: + needs_full_cleanup = True + + if needs_full_cleanup: + print(f"Performing full cleanup ({listener_count} listeners)...") + self.monitor.observe_data_pool( self.data ) + else: + # Efficient cleanup: just reset references without full listener cleanup + print(f"Efficient cleanup ({listener_count} listeners - skipping full cleanup)") + if hasattr(self.monitor, 'data_pool'): + # Just set to None - listeners will be garbage collected + self.monitor.data_pool = None + + print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s") # set the text mark for hdf logging if self.log.textbuffer.get_mark( "lastdumped" ) is None: From ebc2daf1704648d7794e4c84dcea04ab31ac0cd8 Mon Sep 17 00:00:00 2001 From: Markus Rosenstihl Date: Sun, 12 Jul 2026 22:52:39 +0200 Subject: [PATCH 4/4] preformance improvment when using many data_pool objects --- src/damaris/gui/DamarisGUI.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index a206efb..bf864bf 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -597,22 +597,29 @@ class DamarisGUI: # Check if we have significant listener activity needs_full_cleanup = False listener_count = 0 + data_items = 0 if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None: try: - # Count actual listeners instead of just items + # Count both listeners and data items for analysis with self.monitor.data_pool.__dictlock: listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0 + data_items = len(self.monitor.data_pool) needs_full_cleanup = listener_count > 5 # Only full cleanup if many listeners + + # Analyze the structure + flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key) + nested_keys = data_items - flat_keys + print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners") except: needs_full_cleanup = True if needs_full_cleanup: - print(f"Performing full cleanup ({listener_count} listeners)...") + print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...") self.monitor.observe_data_pool( self.data ) else: # Efficient cleanup: just reset references without full listener cleanup - print(f"Efficient cleanup ({listener_count} listeners - skipping full cleanup)") + print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)") if hasattr(self.monitor, 'data_pool'): # Just set to None - listeners will be garbage collected self.monitor.data_pool = None