merged sequential writing

Merge branch 'main' of gitea.pkm.physik.tu-darmstadt.de:IPKM/python3-damaris

.gitignore

@@ -6,3 +6,6 @@ __pycache__
 *.egg-info
 *.whl
 .idea
+debian/files
+.pybuild
+.junie

MANIFEST.in

@@ -1,6 +1,6 @@
-include src/gui/DAMARIS3.png
-include src/gui/DAMARIS3.ico
-include src/gui/damaris.glade
-include src/gui/damaris.gladep
+include src/damaris/gui/DAMARIS3.png
+include src/damaris/gui/DAMARIS3.ico
+include src/damaris/gui/damaris3.glade
+include src/damaris/gui/python.xml
 #recursive-include doc/reference-html *.html *.gif *.png *.css
 #recursive-include doc/tutorial-html *.html *.gif *.png *.css *.tar.gz *.sh

debian/files

@@ -1,2 +0,0 @@
-python3-damaris_0.20_all.deb science optional
-python3-damaris_0.20_amd64.buildinfo science optional

pyproject.toml

@@ -39,7 +39,7 @@ DAMARIS3 = "damaris.__main__:main"
 where = ["src"]
 
 [tool.setuptools.package-data]
-"damaris.gui" = ["DAMARIS3.png", "DAMARIS3.ico", "damaris.xml", "python.xml", "damaris3.glade"]
+"damaris.gui" = ["DAMARIS3.png", "DAMARIS3.ico", "python.xml", "damaris3.glade"]
 
 [tool.setuptools.dynamic]
 version = {attr = "damaris.__version__"}

src/damaris/data/Accumulation.py

@@ -989,14 +989,14 @@ class Accumulation(Errorable, Drawable, DamarisFFT, Signalpath):
                 self.lock.acquire()
 
                 if self.sampling_rate != other.get_sampling_rate():
-                    raise ValueError("Accumulation: You cant add ADC-Results with diffrent sampling-rates")
+                    raise ValueError("Accumulation: You can't add ADC-Results with different sampling-rates")
                 if len(self.y[0]) != len(other):
-                    raise ValueError("Accumulation: You cant add ADC-Results with diffrent number of samples")
+                    raise ValueError("Accumulation: You can't add ADC-Results with different number of samples")
                 if len(self.y) != other.get_number_of_channels():
-                    raise ValueError("Accumulation: You cant add ADC-Results with diffrent number of channels")
+                    raise ValueError("Accumulation: You can't add ADC-Results with different number of channels")
                 for i in range(len(self.index)):
                     if self.index[i] != other.get_index_bounds(i):
-                        raise ValueError("Accumulation: You cant add ADC-Results with diffrent indexing")
+                        raise ValueError("Accumulation: You can't add ADC-Results with different indexing")
 
                 for i in range(self.get_number_of_channels()):
                     self.y[i] += other.y[i]

src/damaris/data/DataPool.py

@@ -49,6 +49,7 @@ class DataPool(collections.abc.MutableMapping):
         self.__mydict={}
         self.__dictlock=threading.Lock()
         self.__registered_listeners=[]
+        self.__dirty_keys=set()
 
     def __getitem__(self, name):
         try:
@@ -65,6 +66,7 @@ class DataPool(collections.abc.MutableMapping):
             else:
                 e=DataPool.Event(DataPool.Event.new_key, name,self)
             self.__mydict[name]=value
+            self.__dirty_keys.add(name)
         finally:
             self.__dictlock.release()
         self.__send_event(e)
@@ -74,6 +76,7 @@ class DataPool(collections.abc.MutableMapping):
         try:
             self.__dictlock.acquire()
             del self.__mydict[name]
+            self.__dirty_keys.add(name)
         finally:
             self.__dictlock.release()
         self.__send_event(DataPool.Event(DataPool.Event.deleted_key,name,self))
@@ -107,7 +110,54 @@ class DataPool(collections.abc.MutableMapping):
         self.__send_event(DataPool.Event(DataPool.Event.destroy))
         self.__registered_listeners=None
 
-    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None):
+    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None, incremental=False):
+
+        """
+        Writes the content of a DataPool object to an HDF5 file using the `tables` library.
+
+        This method serializes the entire content of the data pool into an HDF5 file, creating
+        appropriate groups and arrays based on the keys and values stored in the data pool.
+        Nested storage paths are supported, and group or array names are
+        converted into valid HDF5 names. Paths are prepended with "dir_", results with "dict_".
+        If duplicate names arise, extensions (like `_001`, `_002`)
+        are appended to ensure uniqueness.
+
+        Parameters:
+        hdffile : str | bytes | tables.File
+            Path to the HDF5 file or an already open `tables.File` object where the
+            data should be written. Strings or bytes are interpreted as file paths.
+
+        where : str, optional
+            The root HDF5 group path under which the data pool will be stored.
+            Defaults to "/".
+
+        name : str, optional
+            The name of the root HDF5 group under which the data pool is saved.
+            Defaults to "data_pool".
+
+        complib : str, optional
+            Compression library to be used when creating HDF5 arrays. Optional, defaults
+            to None, meaning no compression is applied.
+
+        complevel : int, optional
+            Compression level to use when applying compression. Has no effect if `complib`
+            is set to None.
+
+        Raises:
+        Exception
+            If `hdffile` is neither a string nor a valid `tables.File` object.
+
+        Notes:
+        - This method assumes that the content of the data pool does not change during the
+          write operation.
+        - Keys in the data pool starting with "__" (double underscores) are skipped.
+        - If a key contains nested paths (e.g., "group1/subgroup2/item"), intermediate
+          groups are created as necessary.
+        - Group or array names are converted to valid HDF5 names, and name conflicts
+          are resolved by appending extensions (e.g., "_001").
+        - Errors during the writing process are logged to the console, and traceback
+          information is displayed for debugging purposes.
+        """
         if isinstance(hdffile, (bytes, str)):
             dump_file=tables.open_file(hdffile, mode="a")
         elif isinstance(hdffile,tables.File):
@@ -115,10 +165,19 @@ class DataPool(collections.abc.MutableMapping):
         else:
             raise Exception("expecting hdffile or string")
 
-        dump_group=dump_file.create_group(where, name, "DAMARIS data pool")
+        if where in dump_file and name in dump_file.get_node(where):
+            dump_group = dump_file.get_node(where, name)
+        else:
+            dump_group = dump_file.create_group(where, name, "DAMARIS data pool")
+
         self.__dictlock.acquire()
-        dict_keys=list(self.__mydict.keys())
+        if incremental:
+            dict_keys=list(self.__dirty_keys)
+        else:
+            # Full sync: current keys + any other dirty keys (e.g. deletions)
+            dict_keys=list(set(self.__mydict.keys()) | self.__dirty_keys)
         self.__dictlock.release()
+
         try:
             for key in dict_keys:
                 if key.startswith("__"):
@@ -143,18 +202,34 @@ class DataPool(collections.abc.MutableMapping):
 
                 # convert last part of key to a valid name
                 group_keyname="dict_"+str(namelist[-1]).translate(DataPool.translation_table)
-                # avoid double names by adding number extension
+
+                # find existing node for this key
                 if group_keyname in dump_dir:
-                    extension_count=0
-                    while group_keyname+"_%03d"%extension_count in dump_dir:
-                        extension_count+=1
-                    group_keyname+="_%03d"%extension_count
+                    if dump_dir._v_children[group_keyname]._v_title == key:
+                        dump_file.remove_node(dump_dir, group_keyname, recursive=True)
+                    else:
+                        extension_count = 0
+                        while group_keyname + "_%03d" % extension_count in dump_dir:
+                            suffix_name = group_keyname + "_%03d" % extension_count
+                            if dump_dir._v_children[suffix_name]._v_title == key:
+                                dump_file.remove_node(dump_dir, suffix_name, recursive=True)
+                                group_keyname = suffix_name
+                                break
+                            extension_count += 1
+                        else:
+                            # truly new key with collision
+                            group_keyname += "_%03d" % extension_count
+
                 self.__dictlock.acquire()
                 if key not in self.__mydict:
-                    # outdated ...
+                    # Key was deleted
+                    if key in self.__dirty_keys:
+                        self.__dirty_keys.remove(key)
                     self.__dictlock.release()
                     continue
                 value=self.__mydict[key]
+                if key in self.__dirty_keys:
+                    self.__dirty_keys.remove(key)
                 self.__dictlock.release()
                 # now write data, assuming, the object is constant during write operation
                 if hasattr(value, "write_to_hdf"):
@@ -194,7 +269,7 @@ class DataPool(collections.abc.MutableMapping):
 
     def read_from_hdf(self, hdffile, where="/data_pool"):
         """
-        Read data from HDF5 file and populate the DataPool.
+        Read specified data from HDF5 file and populate the DataPool.
         
         Parameters:
         - hdffile: HDF5 file object or filename (bytes/str)
@@ -220,6 +295,11 @@ class DataPool(collections.abc.MutableMapping):
             # Recursively process the group structure
             self._read_hdf_group(data_pool_group, "")
             
+            # Clear dirty keys as they are now in sync with HDF5
+            self.__dictlock.acquire()
+            self.__dirty_keys.clear()
+            self.__dictlock.release()
+            
         finally:
             if close_file:
                 hdf_file.close()
@@ -337,7 +417,7 @@ class DataPool(collections.abc.MutableMapping):
     @classmethod
     def load_hdf5(cls, filename):
         """
-        Load a DAMARIS HDF5 file and return DataPool and metadata.
+        Load a complete DAMARIS HDF5 file and return DataPool and metadata, including scripts.
         
         Parameters:
         - filename: Path to HDF5 file

src/damaris/data/MeasurementResult.py

@@ -193,7 +193,7 @@ class MeasurementResult(Drawable.Drawable, collections.UserDict):
 
     def uses_statistics(self):
         """
-        drawable interface method, returns True
+        drawable interface method returns True
         """
         return True
 
@@ -203,10 +203,11 @@ class MeasurementResult(Drawable.Drawable, collections.UserDict):
         destination can be a file or a filename
         suitable for further processing
         """
-        # write sorted
         the_destination=destination
+        file_opened = False
         if type(destination) in (str,):
             the_destination=open(destination, "w")
+            file_opened = True
 
         the_destination.write("# quantity:"+str(self.quantity_name)+"\n")
         the_destination.write("# x y ysigma n\n")
@@ -222,7 +223,8 @@ class MeasurementResult(Drawable.Drawable, collections.UserDict):
                                                           y.mean_error(),
                                                           delimiter,
                                                           y.n))
-        the_destination=None
+        if file_opened:
+            the_destination.close()
 
 
     def write_to_hdf(self, hdffile, where, name, title, complib=None, complevel=None):

src/damaris/data/__init__.py

@@ -6,5 +6,5 @@ from damaris.data.Error_Result import Error_Result
 from damaris.data.Config_Result import Config_Result
 from damaris.data.Temperature import TemperatureResult
 
-__all__=["ADC_Result", "Accumulation", "MeasurementResult", "AccumulatedValue", "DataPool", "FFT", "Error_Result", "Config_Result", "TemperatureResult" ]
+__all__=["ADC_Result", "Accumulation", "MeasurementResult", "AccumulatedValue", "DataPool", "Error_Result", "Config_Result", "TemperatureResult" ]
 

src/damaris/gui/DamarisGUI.py

@@ -88,7 +88,6 @@ class logstream:
     def __del__( self ):
         pass
 
-
 global log
 log = logstream( )
 sys.stdout = log
@@ -134,9 +133,6 @@ class LockFile:
             return False
 
 
-
-
-
 class DamarisGUI:
     ExpScript_Display = 0
     ResScript_Display = 1
@@ -878,15 +874,6 @@ class DamarisGUI:
                                                    filters=tables.Filters( complevel=9, complib='zlib' ) )
 
         if dump_file is None and os.path.isfile( self.dump_filename ) and tables.is_pytables_file( self.dump_filename ):
-            # take some data from dump file and repack
-            os.rename( self.dump_filename, self.dump_filename + ".bak" )
-            old_dump_file = tables.open_file( self.dump_filename + ".bak", mode="r+" )
-            if "data_pool" in old_dump_file.root:
-                old_dump_file.remove_node( where="/", name="data_pool", recursive=True )
-            old_dump_file.copy_file( self.dump_filename )
-            old_dump_file.close( )
-            del old_dump_file
-            os.remove( self.dump_filename + ".bak" )
             # prepare for update
             dump_file = tables.open_file( self.dump_filename, mode="r+" )
 
@@ -903,7 +890,8 @@ class DamarisGUI:
 
         # save the data!
         self.data.write_hdf5( dump_file, where="/", name="data_pool",
-                              complib=self.dump_complib, complevel=self.dump_complevel )
+                              complib=self.dump_complib, complevel=self.dump_complevel,
+                              incremental=not init )
 
         # now save additional information
         e = self.si.data.get( "__recentexperiment", -1 ) + 1
@@ -1184,7 +1172,7 @@ get_ylabel set_ylabel"""
 
         self.compwgen = gtksourceview2.CompletionWords.new("Keywords", None)
         self.compwgen.register(self.keywordsbuffer)
-        self.compwgen.set_property("minimum-word-size", 1)
+        self.compwgen.set_property("minimum-word-size", 2)
 
         self.compwexp = gtksourceview2.CompletionWords.new("Words", None)
         self.compwexp.register(self.experiment_script_textbuffer)
@@ -1698,15 +1686,6 @@ get_ylabel set_ylabel"""
                         if exp_script or res_script:
                             script_widget.set_scripts(exp_script, res_script)
                     
-                    # Show success message
-                    success_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
-                                                    type=gtk.MessageType.INFO,
-                                                    flags=gtk.DialogFlags.MODAL,
-                                                    buttons=("_OK", gtk.ResponseType.OK))
-                    success_dialog.set_markup("Successfully loaded data from %s" % os.path.basename(file_name))
-                    success_dialog.run()
-                    success_dialog.destroy()
-                    
                 except Exception as e:
                     error_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
                                                  type=gtk.MessageType.ERROR,
@@ -2455,7 +2434,9 @@ class MonitorWidgets:
         # now let's add a button to the toolbar
         self.matplot_cursor_button = gtk.ToggleButton(label='')
         try:
-            image = gtk.Image.new_from_file("/usr/share/icons/breeze/actions/24/crosshairs-symbolic.svg")
+            icon_theme = gtk.IconTheme.get_default()
+            icon = icon_theme.load_icon("crosshairs", 24, 0)
+            image = gtk.Image.new_from_pixbuf(icon)
             self.matplot_cursor_button.set_image(image)
         except:
             pass
@@ -2935,6 +2916,7 @@ class MonitorWidgets:
                 self.matplot_cursor = matplotlib.widgets.Cursor(self.matplot_axes,
                                                                 useblit=False, color='red', linewidth=1)
                 self.matplot_canvas.draw()
+            # TODO: Deactivate the pane button
 
         else:
             # Deactivate cursor
@@ -3062,7 +3044,7 @@ class MonitorWidgets:
 
         x_scale = self.display_x_scaling_combobox.get_active_text()
         y_scale = self.display_y_scaling_combobox.get_active_text()
-        print(x_scale)
+
         if x_scale == "lin":
             self.matplot_axes.set_xscale("linear")
         elif x_scale == "log10":

src/damaris/gui/damaris.gladep

@@ -1,8 +0,0 @@
-<?xml version="1.0" standalone="no"?> <!--*- mode: xml -*-->
-<!DOCTYPE glade-project SYSTEM "http://glade.gnome.org/glade-project-2.0.dtd">
-
-<glade-project>
-  <name>damaris-gui</name>
-  <program_name>damaris</program_name>
-  <gnome_support>FALSE</gnome_support>
-</glade-project>

tests/res_test.py

@@ -18,7 +18,6 @@ def result():
         data["test"] = r
         accu+=ts
     accu.write_to_hdf(h5, "/", "name", "title")
-    print(accu)
     try:
         h5.close()
     except: