diff --git a/src/damaris/data/DataPool.py b/src/damaris/data/DataPool.py index 797b023..ac3a4d8 100644 --- a/src/damaris/data/DataPool.py +++ b/src/damaris/data/DataPool.py @@ -49,6 +49,7 @@ class DataPool(collections.abc.MutableMapping): self.__mydict={} self.__dictlock=threading.Lock() self.__registered_listeners=[] + self.__dirty_keys=set() def __getitem__(self, name): try: @@ -65,6 +66,7 @@ class DataPool(collections.abc.MutableMapping): else: e=DataPool.Event(DataPool.Event.new_key, name,self) self.__mydict[name]=value + self.__dirty_keys.add(name) finally: self.__dictlock.release() self.__send_event(e) @@ -74,6 +76,7 @@ class DataPool(collections.abc.MutableMapping): try: self.__dictlock.acquire() del self.__mydict[name] + self.__dirty_keys.add(name) finally: self.__dictlock.release() self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self)) @@ -107,7 +110,8 @@ class DataPool(collections.abc.MutableMapping): self.__send_event(DataPool.Event(DataPool.Event.destroy)) self.__registered_listeners=None - def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None): + def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False): + """ Writes the content of a DataPool object to an HDF5 file using the `tables` library. @@ -161,10 +165,19 @@ class DataPool(collections.abc.MutableMapping): else: raise Exception("expecting hdffile or string") - dump_group=dump_file.create_group(where, name, "DAMARIS data pool") + if where in dump_file and name in dump_file.get_node(where): + dump_group = dump_file.get_node(where, name) + else: + dump_group = dump_file.create_group(where, name, "DAMARIS data pool") + self.__dictlock.acquire() - dict_keys=list(self.__mydict.keys()) + if incremental: + dict_keys=list(self.__dirty_keys) + else: + # Full sync: current keys + any other dirty keys (e.g. deletions) + dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys) self.__dictlock.release() + try: for key in dict_keys: if key.startswith("__"): @@ -189,18 +202,34 @@ class DataPool(collections.abc.MutableMapping): # convert last part of key to a valid name group_keyname="dict_"+str(namelist[-1]).translate(DataPool.translation_table) - # avoid double names by adding number extension + + # find existing node for this key if group_keyname in dump_dir: - extension_count=0 - while group_keyname+"_%03d"%extension_count in dump_dir: - extension_count+=1 - group_keyname+="_%03d"%extension_count + if dump_dir._v_children[group_keyname]._v_title == key: + dump_file.remove_node(dump_dir, group_keyname, recursive=True) + else: + extension_count = 0 + while group_keyname + "_%03d" % extension_count in dump_dir: + suffix_name = group_keyname + "_%03d" % extension_count + if dump_dir._v_children[suffix_name]._v_title == key: + dump_file.remove_node(dump_dir, suffix_name, recursive=True) + group_keyname = suffix_name + break + extension_count += 1 + else: + # truly new key with collision + group_keyname += "_%03d" % extension_count + self.__dictlock.acquire() if key not in self.__mydict: - # outdated ... + # Key was deleted + if key in self.__dirty_keys: + self.__dirty_keys.remove(key) self.__dictlock.release() continue value=self.__mydict[key] + if key in self.__dirty_keys: + self.__dirty_keys.remove(key) self.__dictlock.release() # now write data, assuming, the object is constant during write operation if hasattr(value, "write_to_hdf"): @@ -266,6 +295,11 @@ class DataPool(collections.abc.MutableMapping): # Recursively process the group structure self._read_hdf_group(data_pool_group, "") + # Clear dirty keys as they are now in sync with HDF5 + self.__dictlock.acquire() + self.__dirty_keys.clear() + self.__dictlock.release() + finally: if close_file: hdf_file.close() diff --git a/src/damaris/gui/DamarisGUI.py b/src/damaris/gui/DamarisGUI.py index e5724b7..d97e610 100644 --- a/src/damaris/gui/DamarisGUI.py +++ b/src/damaris/gui/DamarisGUI.py @@ -874,15 +874,6 @@ class DamarisGUI: filters=tables.Filters( complevel=9, complib='zlib' ) ) if dump_file is None and os.path.isfile( self.dump_filename ) and tables.is_pytables_file( self.dump_filename ): - # take some data from dump file and repack - os.rename( self.dump_filename, self.dump_filename + ".bak" ) - old_dump_file = tables.open_file( self.dump_filename + ".bak", mode="r+" ) - if "data_pool" in old_dump_file.root: - old_dump_file.remove_node( where="/", name="data_pool", recursive=True ) - old_dump_file.copy_file( self.dump_filename ) - old_dump_file.close( ) - del old_dump_file - os.remove( self.dump_filename + ".bak" ) # prepare for update dump_file = tables.open_file( self.dump_filename, mode="r+" ) @@ -899,7 +890,8 @@ class DamarisGUI: # save the data! self.data.write_hdf5( dump_file, where="/", name="data_pool", - complib=self.dump_complib, complevel=self.dump_complevel ) + complib=self.dump_complib, complevel=self.dump_complevel, + incremental=not init ) # now save additional information e = self.si.data.get( "__recentexperiment", -1 ) + 1