preformance improvement: starting experiment after createing lots of data is now much faster

src/damaris/data/DataPool.py

@@ -47,20 +47,20 @@ class DataPool(collections.abc.MutableMapping):
 
     def __init__(self):
         self.__mydict={}
-        self.__dictlock=threading.Lock()
-        self.__registered_listeners=[]
+        self._dictlock=threading.RLock()
+        self._registered_listeners=[]
         self.__dirty_keys=set()
 
     def __getitem__(self, name):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             return self.__mydict[name]
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
 
     def __setitem__(self, name, value):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             if name in self.__mydict:
                 e=DataPool.Event(DataPool.Event.updated_value,name,self)
             else:
@@ -68,47 +68,47 @@ class DataPool(collections.abc.MutableMapping):
             self.__mydict[name]=value
             self.__dirty_keys.add(name)
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
         self.__send_event(e)
 
 
     def __delitem__(self, name):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             del self.__mydict[name]
             self.__dirty_keys.add(name)
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
         self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self))
 
     def __iter__(self):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             return iter(self.__mydict)
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
 
     def __len__(self):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             return len(self.__mydict)
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
 
     def keys(self):
         try:
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             return list(self.__mydict.keys())
         finally:
-            self.__dictlock.release()
+            self._dictlock.release()
 
     def __send_event(self, _event):
-        for listeners in self.__registered_listeners:
+        for listeners in self._registered_listeners:
             listeners(_event.copy())
 
     def __del__(self):
         self.__send_event(DataPool.Event(DataPool.Event.destroy))
-        self.__registered_listeners=None
+        self._registered_listeners=None
 
     def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False, write_interval=0):
 
@@ -181,23 +181,17 @@ class DataPool(collections.abc.MutableMapping):
         else:
             dump_group = dump_file.create_group(where, name, "DAMARIS data pool")
 
-        self.__dictlock.acquire()
+        self._dictlock.acquire()
         if incremental:
             dict_keys=list(self.__dirty_keys)
         else:
             # Full sync: current keys + any other dirty keys (e.g. deletions)
             dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
+            # Optimization: for periodic writes, we only write dirty keys
+            if write_interval > 0:
+                dict_keys = list(self.__dirty_keys)
         
-        # Apply optimization based on write_interval
-        if write_interval == 0:
-            # No periodic writes: always use incremental writing for better performance
-            dict_keys = list(self.__dirty_keys)
-        elif write_interval > 0 and not incremental:
-            # Periodic writes enabled: only write dirty keys that haven't been written yet
-            # This prevents rewriting objects that were already written in periodic updates
-            dict_keys = list(self.__dirty_keys)
-        
-        self.__dictlock.release()
+        self._dictlock.release()
 
         try:
             for key in dict_keys:
@@ -241,17 +235,17 @@ class DataPool(collections.abc.MutableMapping):
                             # truly new key with collision
                             group_keyname += "_%03d" % extension_count
 
-                self.__dictlock.acquire()
+                self._dictlock.acquire()
                 if key not in self.__mydict:
                     # Key was deleted
                     if key in self.__dirty_keys:
                         self.__dirty_keys.remove(key)
-                    self.__dictlock.release()
+                    self._dictlock.release()
                     continue
                 value=self.__mydict[key]
                 if key in self.__dirty_keys:
                     self.__dirty_keys.remove(key)
-                self.__dictlock.release()
+                self._dictlock.release()
                 # now write data, assuming, the object is constant during write operation
                 if hasattr(value, "write_to_hdf"):
                     try:
@@ -317,9 +311,9 @@ class DataPool(collections.abc.MutableMapping):
             self._read_hdf_group(data_pool_group, "")
             
             # Clear dirty keys as they are now in sync with HDF5
-            self.__dictlock.acquire()
+            self._dictlock.acquire()
             self.__dirty_keys.clear()
-            self.__dictlock.release()
+            self._dictlock.release()
             
         finally:
             if close_file:
@@ -528,8 +522,8 @@ class DataPool(collections.abc.MutableMapping):
                 hdf_file.close()
 
     def register_listener(self, listening_function):
-        self.__registered_listeners.append(listening_function)
+        self._registered_listeners.append(listening_function)
 
     def unregister_listener(self, listening_function):
-        if listening_function in self.__registered_listeners:
-            self.__registered_listeners.remove(listening_function)
+        if listening_function in self._registered_listeners:
+            self._registered_listeners.remove(listening_function)