implemented incremental HDF5 writing (Fixes #33)

src/damaris/data/DataPool.py

@@ -49,6 +49,7 @@ class DataPool(collections.abc.MutableMapping):
         self.__mydict={}
         self.__dictlock=threading.Lock()
         self.__registered_listeners=[]
+        self.__dirty_keys=set()
 
     def __getitem__(self, name):
         try:
@@ -65,6 +66,7 @@ class DataPool(collections.abc.MutableMapping):
             else:
                 e=DataPool.Event(DataPool.Event.new_key, name,self)
             self.__mydict[name]=value
+            self.__dirty_keys.add(name)
         finally:
             self.__dictlock.release()
         self.__send_event(e)
@@ -74,6 +76,7 @@ class DataPool(collections.abc.MutableMapping):
         try:
             self.__dictlock.acquire()
             del self.__mydict[name]
+            self.__dirty_keys.add(name)
         finally:
             self.__dictlock.release()
         self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self))
@@ -107,7 +110,7 @@ class DataPool(collections.abc.MutableMapping):
         self.__send_event(DataPool.Event(DataPool.Event.destroy))
         self.__registered_listeners=None
 
-    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None):
+    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False):
         if isinstance(hdffile, (bytes, str)):
             dump_file=tables.open_file(hdffile, mode="a")
         elif isinstance(hdffile,tables.File):
@@ -115,10 +118,19 @@ class DataPool(collections.abc.MutableMapping):
         else:
             raise Exception("expecting hdffile or string")
 
-        dump_group=dump_file.create_group(where, name, "DAMARIS data pool")
+        if where in dump_file and name in dump_file.get_node(where):
+            dump_group = dump_file.get_node(where, name)
+        else:
+            dump_group = dump_file.create_group(where, name, "DAMARIS data pool")
+
         self.__dictlock.acquire()
-        dict_keys=list(self.__mydict.keys())
+        if incremental:
+            dict_keys=list(self.__dirty_keys)
+        else:
+            # Full sync: current keys + any other dirty keys (e.g. deletions)
+            dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
         self.__dictlock.release()
+
         try:
             for key in dict_keys:
                 if key.startswith("__"):
@@ -143,18 +155,34 @@ class DataPool(collections.abc.MutableMapping):
 
                 # convert last part of key to a valid name
                 group_keyname="dict_"+str(namelist[-1]).translate(DataPool.translation_table)
-                # avoid double names by adding number extension
+
+                # find existing node for this key
                 if group_keyname in dump_dir:
-                    extension_count=0
+                    if dump_dir._v_children[group_keyname]._v_title == key:
-                    while group_keyname+"_%03d"%extension_count in dump_dir:
+                        dump_file.remove_node(dump_dir, group_keyname, recursive=True)
-                        extension_count+=1
+                    else:
-                    group_keyname+="_%03d"%extension_count
+                        extension_count = 0
+                        while group_keyname + "_%03d" % extension_count in dump_dir:
+                            suffix_name = group_keyname + "_%03d" % extension_count
+                            if dump_dir._v_children[suffix_name]._v_title == key:
+                                dump_file.remove_node(dump_dir, suffix_name, recursive=True)
+                                group_keyname = suffix_name
+                                break
+                            extension_count += 1
+                        else:
+                            # truly new key with collision
+                            group_keyname += "_%03d" % extension_count
+
                 self.__dictlock.acquire()
                 if key not in self.__mydict:
-                    # outdated ...
+                    # Key was deleted
+                    if key in self.__dirty_keys:
+                        self.__dirty_keys.remove(key)
                     self.__dictlock.release()
                     continue
                 value=self.__mydict[key]
+                if key in self.__dirty_keys:
+                    self.__dirty_keys.remove(key)
                 self.__dictlock.release()
                 # now write data, assuming, the object is constant during write operation
                 if hasattr(value, "write_to_hdf"):
@@ -220,6 +248,11 @@ class DataPool(collections.abc.MutableMapping):
             # Recursively process the group structure
             self._read_hdf_group(data_pool_group, "")
             
+            # Clear dirty keys as they are now in sync with HDF5
+            self.__dictlock.acquire()
+            self.__dirty_keys.clear()
+            self.__dictlock.release()
+            
         finally:
             if close_file:
                 hdf_file.close()

src/damaris/gui/DamarisGUI.py

@@ -874,15 +874,6 @@ class DamarisGUI:
                                                    filters=tables.Filters( complevel=9, complib='zlib' ) )
 
         if dump_file is None and os.path.isfile( self.dump_filename ) and tables.is_pytables_file( self.dump_filename ):
-            # take some data from dump file and repack
-            os.rename( self.dump_filename, self.dump_filename + ".bak" )
-            old_dump_file = tables.open_file( self.dump_filename + ".bak", mode="r+" )
-            if "data_pool" in old_dump_file.root:
-                old_dump_file.remove_node( where="/", name="data_pool", recursive=True )
-            old_dump_file.copy_file( self.dump_filename )
-            old_dump_file.close( )
-            del old_dump_file
-            os.remove( self.dump_filename + ".bak" )
             # prepare for update
             dump_file = tables.open_file( self.dump_filename, mode="r+" )
 
@@ -899,7 +890,8 @@ class DamarisGUI:
 
         # save the data!
         self.data.write_hdf5( dump_file, where="/", name="data_pool",
-                              complib=self.dump_complib, complevel=self.dump_complevel )
+                              complib=self.dump_complib, complevel=self.dump_complevel,
+                              incremental=not init )
 
         # now save additional information
         e = self.si.data.get( "__recentexperiment", -1 ) + 1