improved performance a lot, incremental writing of HDF5

src/damaris/data/DataPool.py

@@ -148,10 +148,7 @@ class DataPool(collections.abc.MutableMapping):
             If False, write all keys. Defaults to False.
 
         write_interval : int, optional
-            Time interval in seconds between periodic writes. If 0, no periodic writes are performed.
-            This parameter affects the writing strategy:
-            - If write_interval is 0: Always use incremental writing for better performance
-            - If write_interval > 0: Only write dirty keys that haven't been written yet
+            Time interval in seconds between periodic writes.
             Defaults to 0.
 
         Raises:
@@ -187,9 +184,6 @@ class DataPool(collections.abc.MutableMapping):
         else:
             # Full sync: current keys + any other dirty keys (e.g. deletions)
             dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
-            # Optimization: for periodic writes, we only write dirty keys
-            if write_interval > 0:
-                dict_keys = list(self.__dirty_keys)
         
         self._dictlock.release()
 
@@ -219,22 +213,26 @@ class DataPool(collections.abc.MutableMapping):
                 group_keyname="dict_"+str(namelist[-1]).translate(DataPool.translation_table)
 
                 # find existing node for this key
+                node_found = False
                 if group_keyname in dump_dir:
                     if dump_dir._v_children[group_keyname]._v_title == key:
-                        dump_file.remove_node(dump_dir, group_keyname, recursive=True)
+                        node_found = True
                     else:
                         extension_count = 0
                         while group_keyname + "_%03d" % extension_count in dump_dir:
                             suffix_name = group_keyname + "_%03d" % extension_count
                             if dump_dir._v_children[suffix_name]._v_title == key:
-                                dump_file.remove_node(dump_dir, suffix_name, recursive=True)
                                 group_keyname = suffix_name
+                                node_found = True
                                 break
                             extension_count += 1
                         else:
                             # truly new key with collision
                             group_keyname += "_%03d" % extension_count
 
+                if node_found:
+                    dump_file.remove_node(dump_dir, group_keyname, recursive=True)
+
                 self._dictlock.acquire()
                 if key not in self.__mydict:
                     # Key was deleted
@@ -243,10 +241,12 @@ class DataPool(collections.abc.MutableMapping):
                     self._dictlock.release()
                     continue
                 value=self.__mydict[key]
-                if key in self.__dirty_keys:
-                    self.__dirty_keys.remove(key)
+                # We will clear the dirty flag only after successful write
+                # to handle potential transient failures or re-dirtied items
                 self._dictlock.release()
+                
                 # now write data, assuming, the object is constant during write operation
+                write_success = False
                 if hasattr(value, "write_to_hdf"):
                     try:
                         value.write_to_hdf(hdffile=dump_file,
@@ -255,6 +255,7 @@ class DataPool(collections.abc.MutableMapping):
                                            title=key,
                                            complib=complib,
                                            complevel=complevel)
+                        write_success = True
                     except Exception as e:
                         print("failed to write data_pool[\"%s\"]: %s"%(key,str(e)))
                         traceback_file=io.StringIO()
@@ -270,12 +271,24 @@ class DataPool(collections.abc.MutableMapping):
                                               name=group_keyname,
                                               obj=obj_to_save,
                                               title=key)
+                        write_success = True
                     except Exception as e:
                         print("failed to write data_pool[\"%s\"]: %s"%(key,str(e)))
                 else:
                     print("don't know how to store data_pool[\"%s\"] (type: %s)"%(key, type(value)))
+
+                if write_success:
+                    self._dictlock.acquire()
+                    if key in self.__dirty_keys:
+                        # Only remove if it hasn't been modified again in the meantime
+                        # (Checking identity is a simple way for objects)
+                        if self.__mydict.get(key) is value:
+                            self.__dirty_keys.remove(key)
+                    self._dictlock.release()
                 value=None
 
+            dump_file.flush()
+
         finally:
             dump_group=None
             if isinstance(hdffile, (bytes, str)):