diff --git a/src/damaris/data/DataPool.py b/src/damaris/data/DataPool.py index c454dd4..6a0c73a 100644 --- a/src/damaris/data/DataPool.py +++ b/src/damaris/data/DataPool.py @@ -110,7 +110,7 @@ class DataPool(collections.abc.MutableMapping): self.__send_event(DataPool.Event(DataPool.Event.destroy)) self.__registered_listeners=None - def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False): + def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0): """ Writes the content of a DataPool object to an HDF5 file using the `tables` library. @@ -143,6 +143,17 @@ class DataPool(collections.abc.MutableMapping): Compression level to use when applying compression. Has no effect if `complib` is set to None. + incremental : bool, optional + If True, only write keys that have been modified since the last write (dirty keys). + If False, write all keys. Defaults to False. + + write_interval : int, optional + Time interval in seconds between periodic writes. If 0, no periodic writes are performed. + This parameter affects the writing strategy: + - If write_interval is 0: Always use incremental writing for better performance + - If write_interval > 0: Only write dirty keys that haven't been written yet + Defaults to 0. + Raises: Exception If `hdffile` is neither a string nor a valid `tables.File` object. @@ -176,6 +187,16 @@ class DataPool(collections.abc.MutableMapping): else: # Full sync: current keys + any other dirty keys (e.g. deletions) dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys) + + # Apply optimization based on write_interval + if write_interval == 0: + # No periodic writes: always use incremental writing for better performance + dict_keys = list(self.__dirty_keys) + elif write_interval > 0 and not incremental: + # Periodic writes enabled: only write dirty keys that haven't been written yet + # This prevents rewriting objects that were already written in periodic updates + dict_keys = list(self.__dirty_keys) + self.__dictlock.release() try: diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index 82f486c..7fa2a70 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -501,6 +501,9 @@ class DamarisGUI: dialog.destroy() def start_experiment( self, widget, data=None ): + import time + overall_start = time.time() + print(f"DEBUG: start_experiment called at {overall_start}") # something running? if self.si is not None: @@ -508,11 +511,15 @@ class DamarisGUI: self.si = None # get config values: + config_start = time.time() actual_config = self.config.get( ) + print(f"DEBUG: Config load took {time.time() - config_start:.3f}s") # get scripts and start script interface + gui_start = time.time() self.sw.disable_editing( ) exp_script, res_script = self.sw.get_scripts( ) + print(f"DEBUG: GUI operations took {time.time() - gui_start:.3f}s") if not actual_config[ "start_result_script" ]: res_script = "" if not actual_config[ "start_experiment_script" ]: @@ -528,6 +535,7 @@ class DamarisGUI: # check whether scripts are syntactically valid # should be merged with check script function + compile_start = time.time() exp_code = None if exp_script != "": try: @@ -547,6 +555,7 @@ class DamarisGUI: res_code = None if res_script != "": + # Continue with result script compilation try: res_code = compile( res_script, "Result Script", "exec" ) except SyntaxError as e: @@ -563,6 +572,8 @@ class DamarisGUI: pass print(e) + print(f"DEBUG: Script compilation took {time.time() - compile_start:.3f}s") + # detect error if (exp_script != "" and exp_code is None) or \ (res_script != "" and res_code is None): @@ -590,7 +601,16 @@ class DamarisGUI: interval = 0.1 self.stop_experiment_flag.clear() + # Add timing debug info + import time + start_time = time.time() + print(f"DEBUG: Starting experiment at {start_time}") + self.id = self.lockfile.add_experiment() + print(f"DEBUG: Lock file operation took {time.time() - start_time:.3f}s") + + # Add more timing points + lock_wait_start = time.time() while not self.lockfile.am_i_next(): while gtk.events_pending(): gtk.main_iteration_do(False) @@ -601,7 +621,9 @@ class DamarisGUI: loop_run = 0 loop_run += interval - if self.stop_experiment_flag.isSet(): + print(f"DEBUG: Lock wait took {time.time() - lock_wait_start:.3f}s") + + if self.stop_experiment_flag.isSet(): #Experiment has been stopped, clean up and leave self.experiment_script_statusbar_label.set_text("Experiment stopped") self.state = DamarisGUI.Edit_State @@ -618,19 +640,26 @@ class DamarisGUI: try: self.spool_dir = os.path.abspath( actual_config[ "spool_dir" ] ) # setup script engines + script_interface_start = time.time() self.si = ScriptInterface( exp_code, res_code, backend, self.spool_dir, clear_jobs=actual_config[ "del_jobs_after_execution" ], clear_results=actual_config[ "del_results_after_processing" ] ) + print(f"DEBUG: ScriptInterface creation took {time.time() - script_interface_start:.3f}s") self.data = self.si.data # run frontend and script engines self.monitor.observe_data_pool( self.data ) # Start temperature monitoring if configured (before scripts start) + temp_start = time.time() self.start_temperature_monitoring(actual_config) + print(f"DEBUG: Temperature monitoring startup took {time.time() - temp_start:.3f}s") + + scripts_start = time.time() self.si.runScripts( ) + print(f"DEBUG: Script execution startup took {time.time() - scripts_start:.3f}s") except Exception as e: #print "ToDo evaluate exception",str(e), "at",traceback.extract_tb(sys.exc_info()[2])[-1][1:3] #print "Full traceback:" @@ -807,6 +836,11 @@ class DamarisGUI: self.stop_temperature_monitoring() if self.save_thread is None and self.dump_filename != "": print("all subprocesses ended, saving data pool") + # Disable run button during data saving to prevent UI confusion + gdk.threads_enter() + self.toolbar_run_button.set_sensitive(False) + gdk.threads_leave() + # thread to save data... self.save_thread = threading.Thread( target=self.dump_states, name="dump states" ) self.save_thread.start( ) @@ -1012,7 +1046,13 @@ class DamarisGUI: # save the data! self.data.write_hdf5( dump_file, where="/", name="data_pool", complib=self.dump_complib, complevel=self.dump_complevel, - incremental=not init ) + incremental=not init, write_interval=self.dump_timeinterval ) + + # Re-enable run button if this was called from save thread + if not init and threading.current_thread().name == "dump states": + gdk.threads_enter() + self.toolbar_run_button.set_sensitive(True) + gdk.threads_leave() # now save additional information e = self.si.data.get( "__recentexperiment", -1 ) + 1