Merge performance

Merge branch 'main' of gitea.pkm.physik.tu-darmstadt.de:IPKM/python3-damaris

src/damaris/data/DataPool.py

@@ -110,7 +110,7 @@ class DataPool(collections.abc.MutableMapping):
         self.__send_event(DataPool.Event(DataPool.Event.destroy))
         self.__registered_listeners=None
 
-    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False):
+    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0):
 
         """
         Writes the content of a DataPool object to an HDF5 file using the `tables` library.
@@ -143,6 +143,17 @@ class DataPool(collections.abc.MutableMapping):
             Compression level to use when applying compression. Has no effect if `complib`
             is set to None.
 
+        incremental : bool, optional
+            If True, only write keys that have been modified since the last write (dirty keys).
+            If False, write all keys. Defaults to False.
+
+        write_interval : int, optional
+            Time interval in seconds between periodic writes. If 0, no periodic writes are performed.
+            This parameter affects the writing strategy:
+            - If write_interval is 0: Always use incremental writing for better performance
+            - If write_interval > 0: Only write dirty keys that haven't been written yet
+            Defaults to 0.
+
         Raises:
         Exception
             If `hdffile` is neither a string nor a valid `tables.File` object.
@@ -176,6 +187,16 @@ class DataPool(collections.abc.MutableMapping):
         else:
             # Full sync: current keys + any other dirty keys (e.g. deletions)
             dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
+        
+        # Apply optimization based on write_interval
+        if write_interval == 0:
+            # No periodic writes: always use incremental writing for better performance
+            dict_keys = list(self.__dirty_keys)
+        elif write_interval > 0 and not incremental:
+            # Periodic writes enabled: only write dirty keys that haven't been written yet
+            # This prevents rewriting objects that were already written in periodic updates
+            dict_keys = list(self.__dirty_keys)
+        
         self.__dictlock.release()
 
         try:

src/damaris/gui/DamarisGUI.py

@@ -501,6 +501,9 @@ class DamarisGUI:
         dialog.destroy()
 
     def start_experiment( self, widget, data=None ):
+        import time
+        overall_start = time.time()
+        print(f"DEBUG: start_experiment called at {overall_start}")
 
         # something running?
         if self.si is not None:
@@ -508,11 +511,15 @@ class DamarisGUI:
         self.si = None
 
         # get config values:
+        config_start = time.time()
         actual_config = self.config.get( )
+        print(f"DEBUG: Config load took {time.time() - config_start:.3f}s")
 
         # get scripts and start script interface
+        gui_start = time.time()
         self.sw.disable_editing( )
         exp_script, res_script = self.sw.get_scripts( )
+        print(f"DEBUG: GUI operations took {time.time() - gui_start:.3f}s")
         if not actual_config[ "start_result_script" ]:
             res_script = ""
         if not actual_config[ "start_experiment_script" ]:
@@ -528,6 +535,7 @@ class DamarisGUI:
 
         # check whether scripts are syntactically valid
         # should be merged with check script function
+        compile_start = time.time()
         exp_code = None
         if exp_script != "":
             try:
@@ -547,6 +555,7 @@ class DamarisGUI:
 
         res_code = None
         if res_script != "":
+            # Continue with result script compilation
             try:
                 res_code = compile( res_script, "Result Script", "exec" )
             except SyntaxError as e:
@@ -563,6 +572,8 @@ class DamarisGUI:
                     pass
                 print(e)
 
+        print(f"DEBUG: Script compilation took {time.time() - compile_start:.3f}s")
+
         # detect error
         if (exp_script != "" and exp_code is None) or \
                 (res_script != "" and res_code is None):
@@ -577,9 +588,43 @@ class DamarisGUI:
         self.toolbar_pause_button.set_sensitive( True )
         self.toolbar_pause_button.set_active( False )
 
-        # delete old data
+        # delete old data - use efficient cleanup
+        data_cleanup_start = time.time()
         self.data = None
-        self.monitor.observe_data_pool( self.data )
+        
+        # Modern listener management - only clean up listeners when truly needed
+        if hasattr(self, 'monitor') and self.monitor is not None:
+            # Check if we have significant listener activity
+            needs_full_cleanup = False
+            listener_count = 0
+            data_items = 0
+            
+            if hasattr(self.monitor, 'data_pool') and self.monitor.data_pool is not None:
+                try:
+                    # Count both listeners and data items for analysis
+                    with self.monitor.data_pool.__dictlock:
+                        listener_count = len(self.monitor.data_pool.__registered_listeners) if hasattr(self.monitor.data_pool, '__registered_listeners') else 0
+                        data_items = len(self.monitor.data_pool)
+                    needs_full_cleanup = listener_count > 5  # Only full cleanup if many listeners
+                    
+                    # Analyze the structure
+                    flat_keys = sum(1 for key in self.monitor.data_pool.keys() if '/' not in key)
+                    nested_keys = data_items - flat_keys
+                    print(f"DEBUG: Data pool analysis - {data_items} total items, {flat_keys} flat, {nested_keys} nested, {listener_count} listeners")
+                except:
+                    needs_full_cleanup = True
+            
+            if needs_full_cleanup:
+                print(f"Performing full cleanup ({listener_count} listeners, {data_items} items)...")
+                self.monitor.observe_data_pool( self.data )
+            else:
+                # Efficient cleanup: just reset references without full listener cleanup
+                print(f"Efficient cleanup ({listener_count} listeners, {data_items} items - skipping full cleanup)")
+                if hasattr(self.monitor, 'data_pool'):
+                    # Just set to None - listeners will be garbage collected
+                    self.monitor.data_pool = None
+        
+        print(f"DEBUG: Efficient data cleanup took {time.time() - data_cleanup_start:.3f}s")
 
         # set the text mark for hdf logging
         if self.log.textbuffer.get_mark( "lastdumped" ) is None:
@@ -590,7 +635,16 @@ class DamarisGUI:
         interval = 0.1
         self.stop_experiment_flag.clear()
 
+        # Add timing debug info
+        import time
+        start_time = time.time()
+        print(f"DEBUG: Starting experiment at {start_time}")
+
         self.id = self.lockfile.add_experiment()
+        print(f"DEBUG: Lock file operation took {time.time() - start_time:.3f}s")
+        
+        # Add more timing points
+        lock_wait_start = time.time()
         while not self.lockfile.am_i_next():
             while gtk.events_pending():
                 gtk.main_iteration_do(False)
@@ -601,7 +655,9 @@ class DamarisGUI:
                 loop_run = 0
             loop_run += interval
 
-            if self.stop_experiment_flag.isSet():
+        print(f"DEBUG: Lock wait took {time.time() - lock_wait_start:.3f}s")
+
+        if self.stop_experiment_flag.isSet():
                 #Experiment has been stopped, clean up and leave
                 self.experiment_script_statusbar_label.set_text("Experiment stopped")
                 self.state = DamarisGUI.Edit_State
@@ -618,19 +674,26 @@ class DamarisGUI:
         try:
             self.spool_dir = os.path.abspath( actual_config[ "spool_dir" ] )
             # setup script engines
+            script_interface_start = time.time()
             self.si = ScriptInterface( exp_code,
                                        res_code,
                                        backend,
                                        self.spool_dir,
                                        clear_jobs=actual_config[ "del_jobs_after_execution" ],
                                        clear_results=actual_config[ "del_results_after_processing" ] )
+            print(f"DEBUG: ScriptInterface creation took {time.time() - script_interface_start:.3f}s")
 
             self.data = self.si.data
             # run frontend and script engines
             self.monitor.observe_data_pool( self.data )
             # Start temperature monitoring if configured (before scripts start)
+            temp_start = time.time()
             self.start_temperature_monitoring(actual_config)
+            print(f"DEBUG: Temperature monitoring startup took {time.time() - temp_start:.3f}s")
+            
+            scripts_start = time.time()
             self.si.runScripts( )
+            print(f"DEBUG: Script execution startup took {time.time() - scripts_start:.3f}s")
         except Exception as e:
             #print "ToDo evaluate exception",str(e), "at",traceback.extract_tb(sys.exc_info()[2])[-1][1:3]
             #print "Full traceback:"
@@ -807,6 +870,11 @@ class DamarisGUI:
             self.stop_temperature_monitoring()
             if self.save_thread is None and self.dump_filename != "":
                 print("all subprocesses ended, saving data pool")
+                # Disable run button during data saving to prevent UI confusion
+                gdk.threads_enter()
+                self.toolbar_run_button.set_sensitive(False)
+                gdk.threads_leave()
+                
                 # thread to save data...
                 self.save_thread = threading.Thread( target=self.dump_states, name="dump states" )
                 self.save_thread.start( )
@@ -1012,7 +1080,13 @@ class DamarisGUI:
         # save the data!
         self.data.write_hdf5( dump_file, where="/", name="data_pool",
                               complib=self.dump_complib, complevel=self.dump_complevel,
-                              incremental=not init )
+                              incremental=not init, write_interval=self.dump_timeinterval )
+
+        # Re-enable run button if this was called from save thread
+        if not init and threading.current_thread().name == "dump states":
+            gdk.threads_enter()
+            self.toolbar_run_button.set_sensitive(True)
+            gdk.threads_leave()
 
         # now save additional information
         e = self.si.data.get( "__recentexperiment", -1 ) + 1
@@ -3006,21 +3080,29 @@ class MonitorWidgets:
         register a listener and save reference to data
         assume to be in gtk/gdk lock
         """
+        import time
+        cleanup_start = time.time()
         gdk.threads_enter( )
         try:
             if self.data_pool is not None:
                 # maybe some extra cleanup needed
                 print("ToDo: cleanup widgets")
+                widget_cleanup_start = time.time()
                 if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
                     self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
                     self.displayed_data[ 1 ] = None
                 self.displayed_data = [ None, None ]
+                print(f"DEBUG: Widget cleanup took {time.time() - widget_cleanup_start:.3f}s")
+                
+                listener_cleanup_start = time.time()
                 self.data_pool.unregister_listener( self.datapool_listener )
                 self.data_pool = None
+                print(f"DEBUG: Listener cleanup took {time.time() - listener_cleanup_start:.3f}s")
 
                 self.update_counter_lock.acquire( )
                 self.update_counter = 0
                 self.update_counter_lock.release( )
+                print(f"DEBUG: Total data pool cleanup took {time.time() - cleanup_start:.3f}s")
 
             # display states
             self.source_list_reset( )