added HDF5 read capability. Closes #27

2026-07-06 20:42:04 +02:00
parent c181e15aa8
commit 7ba5c7d56a
3 changed files with 419 additions and 32 deletions
@@ -7,6 +7,7 @@ import collections
 import threading
 import traceback
 import io
+import numpy

 class DataPool(collections.abc.MutableMapping):
    """
@@ -107,7 +108,7 @@ class DataPool(collections.abc.MutableMapping):
        self.__registered_listeners=None

    def write_hdf5(self,hdffile,where="/",name="data_pool", complib=None, complevel=None):
-        if type(hdffile) is bytes:
+        if isinstance(hdffile, (bytes, str)):
            dump_file=tables.open_file(hdffile, mode="a")
        elif isinstance(hdffile,tables.File):
            dump_file=hdffile
@@ -156,7 +157,7 @@ class DataPool(collections.abc.MutableMapping):
                value=self.__mydict[key]
                self.__dictlock.release()
                # now write data, assuming, the object is constant during write operation
-                if "write_to_hdf" in dir(value):
+                if hasattr(value, "write_to_hdf"):
                    try:
                        value.write_to_hdf(hdffile=dump_file,
                                           where=dump_dir,
@@ -170,16 +171,255 @@ class DataPool(collections.abc.MutableMapping):
                        traceback.print_tb(sys.exc_info()[2], None, traceback_file)
                        print("detailed traceback: %s\n"%str(e)+traceback_file.getvalue())
                        traceback_file=None
+                elif isinstance(value, (numpy.ndarray, int, float, str, bytes)):
+                    try:
+                        obj_to_save = value
+                        if isinstance(value, str):
+                            obj_to_save = value.encode('utf-8')
+                        dump_file.create_array(dump_dir,
+                                              name=group_keyname,
+                                              obj=obj_to_save,
+                                              title=key)
+                    except Exception as e:
+                        print("failed to write data_pool[\"%s\"]: %s"%(key,str(e)))
                else:
-                    print("don't know how to store data_pool[\"%s\"]"%key)
+                    print("don't know how to store data_pool[\"%s\"] (type: %s)"%(key, type(value)))
                value=None

        finally:
            dump_group=None
-            if type(hdffile) is bytes:
+            if isinstance(hdffile, (bytes, str)):
                dump_file.close()
            dump_file=None

+    def read_from_hdf(self, hdffile, where="/data_pool"):
+        """
+        Read data from HDF5 file and populate the DataPool.
+        
+        Parameters:
+        - hdffile: HDF5 file object or filename (bytes/str)
+        - where: Location within HDF5 file to read from (default: "/data_pool")
+        """
+        # Handle both filename and file object
+        if type(hdffile) is bytes or isinstance(hdffile, str):
+            hdf_file = tables.open_file(hdffile, mode="r")
+            close_file = True
+        elif isinstance(hdffile, tables.File):
+            hdf_file = hdffile
+            close_file = False
+        else:
+            raise Exception("expecting hdffile or string")
+        
+        try:
+            # Navigate to the data pool group
+            if where not in hdf_file:
+                raise Exception(f"HDF5 location '{where}' not found")
+            
+            data_pool_group = hdf_file.get_node(where)
+            
+            # Recursively process the group structure
+            self._read_hdf_group(data_pool_group, "")
+            
+        finally:
+            if close_file:
+                hdf_file.close()
+
+    def _read_hdf_group(self, group, path_prefix):
+        """
+        Recursively read HDF5 group and populate DataPool.
+        
+        Parameters:
+        - group: HDF5 group object
+        - path_prefix: Current path in DataPool hierarchy
+        """
+        # Process all nodes in this group
+        for node in group:
+            # Get the node name (last part of the path)
+            node_name = node._v_name
+            
+            # Try to get the original name from title
+            title = node._v_title
+            
+            if title:
+                # For DAMARIS objects, the title is usually the full path
+                # For intermediate groups, it's just the part name
+                if "/" in title:
+                    full_path = title
+                else:
+                    if path_prefix:
+                        full_path = f"{path_prefix}/{title}"
+                    else:
+                        full_path = title
+            else:
+                # Strip dir_ and dict_ prefixes from node name
+                clean_name = self._clean_hdf_name(node_name)
+                
+                # Build the full path in DataPool hierarchy
+                if path_prefix:
+                    full_path = f"{path_prefix}/{clean_name}"
+                else:
+                    full_path = clean_name
+            
+            full_path = full_path.replace("//", "/")
+            
+            if isinstance(node, (tables.Group, tables.Table)):
+                # Check if this is a known DAMARIS object type
+                if hasattr(node._v_attrs, 'damaris_type'):
+                    damaris_type = node._v_attrs.damaris_type
+                    
+                    # Handle different DAMARIS object types
+                    if damaris_type == "Accumulation":
+                        from damaris.data.Accumulation import read_from_hdf
+                        obj = read_from_hdf(node)
+                        if obj is not None:
+                            self[full_path] = obj
+                    
+                    elif damaris_type == "ADC_Result":
+                        from damaris.data.ADC_Result import read_from_hdf
+                        obj = read_from_hdf(node)
+                        if obj is not None:
+                            self[full_path] = obj
+                            
+                    elif damaris_type == "MeasurementResult":
+                        from damaris.data.MeasurementResult import read_from_hdf
+                        obj = read_from_hdf(node)
+                        if obj is not None:
+                            self[full_path] = obj
+                    
+                    # Skip further processing of this group since we handled it
+                    continue
+                
+                # Recursively process sub-groups
+                if isinstance(node, tables.Group):
+                    self._read_hdf_group(node, full_path)
+                
+            elif isinstance(node, (tables.Array, tables.CArray, tables.EArray, tables.Table)):
+                # Handle simple array data
+                try:
+                    data = node.read()
+                    # If it's a scalar array of bytes, decode it to string
+                    if hasattr(data, 'decode'):
+                        try:
+                            data = data.decode('utf-8')
+                        except:
+                            pass
+                    elif isinstance(data, numpy.ndarray) and data.dtype.kind in ('S', 'V'):
+                        # For numpy byte arrays, try to decode if they are scalar or small
+                        if data.size == 1:
+                            try:
+                                decoded = data.item().decode('utf-8')
+                                data = decoded
+                            except:
+                                pass
+                    self[full_path] = data
+                except Exception as e:
+                    print(f"Warning: Could not read array {full_path}: {e}")
+            
+            # Note: Other node types (Table, etc.) would be handled here as needed
+    
+    def _clean_hdf_name(self, name):
+        """
+        Clean HDF5 node name by removing dir_ and dict_ prefixes.
+        
+        Parameters:
+        - name: HDF5 node name (may have dir_ or dict_ prefix)
+        
+        Returns:
+        - Cleaned name with prefixes removed
+        """
+        if name.startswith("dir_"):
+            return name[4:]  # Remove "dir_" prefix
+        elif name.startswith("dict_"):
+            return name[5:]  # Remove "dict_" prefix
+        else:
+            return name
+
+    @classmethod
+    def load_hdf5(cls, filename):
+        """
+        Load a DAMARIS HDF5 file and return DataPool and metadata.
+        
+        Parameters:
+        - filename: Path to HDF5 file
+        
+        Returns:
+        - tuple: (DataPool instance, metadata dict)
+        
+        metadata dict contains:
+        - 'scripts': dict with experiment scripts and spool info
+        - 'log': log data if available
+        - 'timeline': timeline data if available
+        """
+        # Validate file
+        if not tables.is_pytables_file(filename):
+            raise Exception(f"File {filename} is not a valid PyTables HDF5 file")
+        
+        # Create DataPool and load data
+        data_pool = cls()
+        
+        try:
+            hdf_file = tables.open_file(filename, mode="r")
+            
+            # Load main data pool
+            if "/data_pool" in hdf_file:
+                data_pool.read_from_hdf(hdf_file, where="/data_pool")
+            
+            # Extract metadata
+            metadata = {}
+            
+            # Load scripts
+            if "/scripts" in hdf_file:
+                scripts_group = hdf_file.get_node("/scripts")
+                scripts_data = {}
+                # Use _v_children to get node names safely
+                if hasattr(scripts_group, '_v_children'):
+                    for node_name, node in scripts_group._v_children.items():
+                        try:
+                            if hasattr(node, 'read'):
+                                data = node.read()
+                                # Handle numpy string arrays and other types
+                                if hasattr(data, 'decode'):
+                                    # bytes or numpy bytes
+                                    scripts_data[node_name] = data.decode('utf-8', errors='replace')
+                                elif hasattr(data, 'tolist'):
+                                    # numpy array
+                                    scripts_data[node_name] = str(data.tolist())
+                                elif isinstance(data, (str, bytes)):
+                                    # string or bytes
+                                    if isinstance(data, bytes):
+                                        scripts_data[node_name] = data.decode('utf-8', errors='replace')
+                                    else:
+                                        scripts_data[node_name] = data
+                                else:
+                                    scripts_data[node_name] = str(data)
+                        except Exception as e:
+                            print(f"Warning: Could not read script {node_name}: {e}")
+                metadata['scripts'] = scripts_data
+            
+            # Load log
+            if "/log" in hdf_file:
+                try:
+                    log_node = hdf_file.get_node("/log")
+                    metadata['log'] = log_node.read()
+                except Exception as e:
+                    print(f"Warning: Could not read log: {e}")
+            
+            # Load timeline
+            if "/timeline" in hdf_file:
+                try:
+                    timeline_node = hdf_file.get_node("/timeline")
+                    metadata['timeline'] = timeline_node.read()
+                except Exception as e:
+                    print(f"Warning: Could not read timeline: {e}")
+            
+            return data_pool, metadata
+            
+        except Exception as e:
+            raise Exception(f"Failed to load HDF5 file {filename}: {e}")
+        finally:
+            if 'hdf_file' in locals():
+                hdf_file.close()
+
    def register_listener(self, listening_function):
        self.__registered_listeners.append(listening_function)

@@ -169,7 +169,7 @@ class DamarisGUI:

        self.log = LogWindow( self.xml_gui , self)

-        self.sw = ScriptWidgets( self.xml_gui )
+        self.sw = ScriptWidgets( self.xml_gui, self )

        self.toolbar_init( )

@@ -218,6 +218,7 @@ class DamarisGUI:
                        "on_experiment_script_line_textfield_value_changed": self.sw.column_line_widgets_changed_event,
                        "on_experiment_script_column_textfield_value_changed": self.sw.column_line_widgets_changed_event,
                        "on_toolbar_open_file_button_clicked": self.sw.open_file,
+                        "on_toolbar_open_hdf5_button_clicked": self.sw.open_hdf5,
                        "on_toolbar_new_button_clicked": self.sw.new_file,
                        "on_toolbar_save_as_button_clicked": self.sw.save_file_as,
                        "on_toolbar_save_file_button_clicked": self.sw.save_file,
@@ -297,6 +298,14 @@ class DamarisGUI:
        self.config = None
        self.xml_gui = None

+    def observe_data_pool( self, data_pool ):
+        """
+        Delegate to monitor widgets
+        """
+        self.data = data_pool
+        if hasattr(self, "monitor"):
+            self.monitor.observe_data_pool(data_pool)
+
    # event handling: the real acitons in gui programming

    # first global events
@@ -1103,11 +1112,12 @@ class LogWindow:


 class ScriptWidgets:
-    def __init__( self, xml_gui ):
+    def __init__( self, xml_gui, outer_space ):
        """
        initialize text widgets with text
        """
        self.xml_gui = xml_gui
+        self.outer_space = outer_space

        # my states
        # editing enabled/disabled
@@ -1572,7 +1582,7 @@ get_ylabel set_ylabel"""
            elif response == gtk.ResponseType.CANCEL:
                return 0

-        def response( self, response_id, script_widget ):
+        def response( dialog, response_id, script_widget ):
            if response_id == gtk.ResponseType.OK:
                file_name = dialog.get_filename( )
                if file_name is None:
@@ -1580,7 +1590,13 @@ get_ylabel set_ylabel"""

                script_filename = os.path.abspath( file_name )
                if not os.access( script_filename, os.R_OK ):
-                    self.outer_space.show_error_dialog( "File I/O Error", "Cannot read from file %s" % script_filename )
+                    error_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
+                                                   type=gtk.MessageType.ERROR,
+                                                   flags=gtk.DialogFlags.MODAL,
+                                                   buttons=("_OK", gtk.ResponseType.OK))
+                    error_dialog.set_markup("Cannot read from file %s" % script_filename)
+                    error_dialog.run()
+                    error_dialog.destroy()
                    return True

                script_string = script_widget.load_file_as_unicode( script_filename )
@@ -1613,6 +1629,103 @@ get_ylabel set_ylabel"""

        return True

+    def open_hdf5( self, widget, Data=None ):
+        """
+        Open and load an HDF5 file written by DAMARIS.
+        """
+        # ignore
+        if not self.editing_state:
+            return 0
+
+        parent_window = self.xml_gui.get_object( "main_window" )
+        dialog = gtk.FileChooserDialog( title="Open HDF5 File...",
+                                        parent=parent_window,
+                                        action=gtk.FileChooserAction.OPEN,
+                                        buttons=("_Open", gtk.ResponseType.OK, "_Cancel", gtk.ResponseType.CANCEL)
+                                        )
+        dialog.set_default_response( gtk.ResponseType.OK )
+        dialog.set_select_multiple( False )
+
+        # Add filter for HDF5 files
+        filter_hdf5 = gtk.FileFilter()
+        filter_hdf5.set_name("HDF5 files")
+        filter_hdf5.add_pattern("*.h5")
+        filter_hdf5.add_pattern("*.hdf5")
+        filter_hdf5.add_pattern("*.HDF5")
+        filter_hdf5.add_pattern("*.H5")
+        dialog.add_filter(filter_hdf5)
+
+        # Also allow all files
+        filter_all = gtk.FileFilter()
+        filter_all.set_name("All files")
+        filter_all.add_pattern("*")
+        dialog.add_filter(filter_all)
+
+        def response( dialog, response_id, script_widget ):
+            if response_id == gtk.ResponseType.OK:
+                file_name = dialog.get_filename( )
+                if file_name is None:
+                    return
+
+                file_name = os.path.abspath( file_name )
+                if not os.access( file_name, os.R_OK ):
+                    error_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
+                                                   type=gtk.MessageType.ERROR,
+                                                   flags=gtk.DialogFlags.MODAL,
+                                                   buttons=("_OK", gtk.ResponseType.OK))
+                    error_dialog.set_markup("Cannot read from file %s" % file_name)
+                    error_dialog.run()
+                    error_dialog.destroy()
+                    return True
+
+                try:
+                    # Load DataPool and metadata from HDF5
+                    data_pool, metadata = DataPool.load_hdf5(file_name)
+
+                    # Set the loaded data pool in the outer space (main application)
+                    if hasattr(script_widget.outer_space, 'observe_data_pool'):
+                        script_widget.outer_space.observe_data_pool(data_pool)
+                    elif hasattr(script_widget.outer_space, 'monitor'):
+                        # Fallback if delegate not present
+                        script_widget.outer_space.data = data_pool
+                        script_widget.outer_space.monitor.observe_data_pool(data_pool)
+                    
+                    # If there are scripts in metadata, load them
+                    if 'scripts' in metadata:
+                        scripts = metadata['scripts']
+                        exp_script = scripts.get('experiment_script', None)
+                        res_script = scripts.get('result_script', None)
+                        if exp_script or res_script:
+                            script_widget.set_scripts(exp_script, res_script)
+                    
+                    # Show success message
+                    success_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
+                                                    type=gtk.MessageType.INFO,
+                                                    flags=gtk.DialogFlags.MODAL,
+                                                    buttons=("_OK", gtk.ResponseType.OK))
+                    success_dialog.set_markup("Successfully loaded data from %s" % os.path.basename(file_name))
+                    success_dialog.run()
+                    success_dialog.destroy()
+                    
+                except Exception as e:
+                    error_dialog = gtk.MessageDialog(parent=script_widget.xml_gui.get_object( "main_window"),
+                                                 type=gtk.MessageType.ERROR,
+                                                 flags=gtk.DialogFlags.MODAL,
+                                                 buttons=("_OK", gtk.ResponseType.OK))
+                    error_dialog.set_markup("Failed to load HDF5 file: %s" % str(e))
+                    error_dialog.run()
+                    error_dialog.destroy()
+
+            return True
+
+        # Event-Handler for response-signal (when one of the button is pressed)
+        dialog.connect( "response", response, self )
+
+        dialog.run( )
+        dialog.destroy( )
+
+        return True
+
    def save_file( self, widget=None, Data=None ):
        """
        save file to associated filename
@@ -2411,7 +2524,7 @@ class MonitorWidgets:
    def source_list_find_one( self, model, iter, what ):
        """find node in subcategory"""
        while iter is not None:
-            if model.get( iter, 0 )[ 0 ] == what:
+            if str( model.get( iter, 0 )[ 0 ] ) == what:
                return iter
            iter = model.iter_next( iter )
        return iter
@@ -2482,31 +2595,50 @@ class MonitorWidgets:
        register a listener and save reference to data
        assume to be in gtk/gdk lock
        """
-        if self.data_pool is not None:
-            # maybe some extra cleanup needed
-            print("ToDo: cleanup widgets")
-            if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
-                self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
-                self.displayed_data[ 1 ] = None
-            self.displayed_data = [ None, None ]
-            self.data_pool.unregister_listener( self.datapool_listener )
-            self.data_pool = None
+        gdk.threads_enter( )
+        try:
+            if self.data_pool is not None:
+                # maybe some extra cleanup needed
+                print("ToDo: cleanup widgets")
+                if self.displayed_data[ 1 ] is not None and hasattr( self.displayed_data[ 1 ], "unregister_listener" ):
+                    self.displayed_data[ 1 ].unregister_listener( self.datastructures_listener )
+                    self.displayed_data[ 1 ] = None
+                self.displayed_data = [ None, None ]
+                self.data_pool.unregister_listener( self.datapool_listener )
+                self.data_pool = None

+                self.update_counter_lock.acquire( )
+                self.update_counter = 0
+                self.update_counter_lock.release( )
+
+            # display states
            self.source_list_reset( )
-            self.update_counter_lock.acquire( )
-            self.update_counter = 0
-            self.update_counter_lock.release( )
+            self.__rescale = True
+            self.displayed_data = [ None, None ]
+            self.display_source_path_label.set_label( "" )
+            self.clear_display( )

-        # display states
-        self.__rescale = True
-        self.displayed_data = [ None, None ]
-        self.display_source_path_label.set_label( "" )
-        self.clear_display( )
+            if data_pool is not None:
+                # keep track of data
+                self.data_pool = data_pool
+                self.data_pool.register_listener( self.datapool_listener )
                
-        if data_pool is not None:
-            # keep track of data
-            self.data_pool = data_pool
-            self.data_pool.register_listener( self.datapool_listener )
+                # Populate source list with existing data since the listener
+                # was registered after the data was loaded
+                for key in sorted(data_pool.keys()):
+                    if key != "__builtins__":  # Skip internal keys
+                        self.source_list_add(key)
+                
+                # Select first data item if available
+                keys = [k for k in data_pool.keys() if not k.startswith("__")]
+                if keys:
+                    keys.sort()
+                    first_key = keys[0]
+                    first_iter = self.source_list_find(first_key.split("/"))
+                    if first_iter:
+                        self.display_source_combobox.set_active_iter(first_iter)
+        finally:
+            gdk.threads_leave( )

    #################### observing data structures and produce idle events

@@ -2643,7 +2775,7 @@ class MonitorWidgets:
            gdk.threads_leave( )
        elif event.what == DataPool.Event.destroy:
            gdk.threads_enter( )
-            self.source_list_reset( 'None' )
+            self.source_list_reset( )
            self.displayed_data = [ None, None ]
            self.clear_display( )
            gdk.threads_leave( )
@@ -491,6 +491,21 @@ Public License instead of this License.
                <property name="homogeneous">True</property>
              </packing>
            </child>
+            <child>
+              <object class="GtkToolButton" id="toolbar_open_hdf5_button">
+                <property name="visible">True</property>
+                <property name="can-focus">False</property>
+                <property name="tooltip-text" translatable="yes">Open DAMARIS HDF5 file</property>
+                <property name="label">Open HDF5</property>
+                <property name="use-underline">True</property>
+                <property name="icon-name">document-open</property>
+                <signal name="clicked" handler="on_toolbar_open_hdf5_button_clicked" swapped="no"/>
+              </object>
+              <packing>
+                <property name="expand">False</property>
+                <property name="homogeneous">True</property>
+              </packing>
+            </child>
            <child>
              <object class="GtkToolButton" id="toolbar_save_file_button">
                <property name="visible">True</property>