From dda8ae7df0bacb243a4cfb70d2a084fd08469731 Mon Sep 17 00:00:00 2001 From: Markus Rosenstihl Date: Wed, 21 Oct 2026 00:48:06 +0200 Subject: [PATCH] script for release added, pyrefly checks are fixed for now --- .gitea/workflows/debian-build.yml | 14 +++- scripts/release.sh | 65 ++++++++++++++++++ src/damaris/data/ADC_Result.py | 58 +++++++---------- src/damaris/data/Accumulation.py | 94 +++++++++++++-------------- src/damaris/data/MeasurementResult.py | 23 ++++--- tests/test_Accumulation.py | 14 ++++ 6 files changed, 179 insertions(+), 89 deletions(-) create mode 100755 scripts/release.sh diff --git a/.gitea/workflows/debian-build.yml b/.gitea/workflows/debian-build.yml index c39f6a8..32e1ef5 100644 --- a/.gitea/workflows/debian-build.yml +++ b/.gitea/workflows/debian-build.yml @@ -22,13 +22,25 @@ jobs: - name: Install Node.js and git run: | apt-get update - apt-get -y install nodejs git lsb-release jq + apt-get -y install nodejs git lsb-release jq devscripts - name: Checkout repository uses: actions/checkout@v3 with: github-server-url: https://gitea.pkm.physik.tu-darmstadt.de + - name: Set package version from release tag + if: startsWith(gitea.ref, 'refs/tags/v') + env: + TAG: ${{ gitea.ref_name }} + run: | + set -euo pipefail + VERSION="${TAG#v}" + if [ "$(dpkg-parsechangelog --show-field Version)" != "$VERSION" ]; then + dch --newversion "$VERSION" --distribution stable --force-distribution --noquery "Release $TAG" + fi + sed -i -E "s/^(__version__ = \")([^\"]+)(\")$/\1${VERSION}\3/" src/damaris/__init__.py + - name: Install build dependencies run: bash requirements.debian diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000..e72d5bc --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + echo "Usage: $0 VERSION (for example: 0.30)" +} + +if [[ ${1:-} == "-h" || ${1:-} == "--help" ]]; then + usage + exit 0 +fi + +if [[ $# -ne 1 || ! $1 =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + usage >&2 + exit 2 +fi + +version=$1 +tag="v$version" +root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +cd "$root" + +if [[ -n $(git status --porcelain) ]]; then + echo "Release requires a clean working tree." >&2 + exit 1 +fi + +if git rev-parse -q --verify "refs/tags/$tag" >/dev/null; then + echo "Tag $tag already exists." >&2 + exit 1 +fi + +command -v dch >/dev/null || { echo "Install devscripts to use dch." >&2; exit 1; } +command -v dpkg-parsechangelog >/dev/null || { echo "Install dpkg-dev to use dpkg-parsechangelog." >&2; exit 1; } + +current_version=$(dpkg-parsechangelog --show-field Version) +if [[ $current_version == "$version" ]]; then + echo "debian/changelog already has version $version." >&2 + exit 1 +fi + +dch --newversion "$version" --distribution stable --force-distribution +sed -i "s/^__version__ = .*/__version__ = \"$version\"/" src/damaris/__init__.py + +if [[ $(dpkg-parsechangelog --show-field Version) != "$version" ]] || + [[ $(sed -n 's/^__version__ = "\([^"]*\)"$/\1/p' src/damaris/__init__.py) != "$version" ]]; then + echo "Release version did not update consistently; inspect the changelog and package version." >&2 + exit 1 +fi + +echo "About to commit the version bump and create tag $tag." +read -r -p "Continue? [y/N] " answer +if [[ ! $answer =~ ^[Yy]$ ]]; then + echo "Release preparation stopped; version and changelog edits were kept." + exit 1 +fi + +git add debian/changelog src/damaris/__init__.py +git commit -m "Release $tag" +git tag -a "$tag" -m "Release $tag" + +echo "Created local release tag $tag. Push the commit and tag to start the package workflow." +echo " git push origin HEAD" +echo " git push origin $tag" \ No newline at end of file diff --git a/src/damaris/data/ADC_Result.py b/src/damaris/data/ADC_Result.py index a56c839..254bee1 100644 --- a/src/damaris/data/ADC_Result.py +++ b/src/damaris/data/ADC_Result.py @@ -12,13 +12,14 @@ import numpy import sys import datetime import tables -from typing import IO +from typing import IO, Any, Literal, cast from scipy.signal import filtfilt, remez _XArray = numpy.ndarray | list _YChannels = list[numpy.ndarray] _IndexList = list[tuple[int, int]] _Description = dict[str, str] +_CompressionLibrary = Literal["blosc", "blosc2", "bzip2", "lzo", "zlib"] ############################################################################# # # # Name: Class ADC_Result # @@ -377,13 +378,14 @@ class ADC_Result(MeasurementData, Resultable, Drawable, DamarisFFT, Signalpath): def write_to_hdf(self, hdffile: tables.File, where: str, name: str, title: str, complib: str | None = None, complevel: int | None = None) -> None: accu_group=hdffile.create_group(where=where,name=name,title=title) - accu_group._v_attrs.damaris_type="ADC_Result" + attrs = cast(Any, accu_group._v_attrs) + attrs.damaris_type="ADC_Result" if self.contains_data(): self.lock.acquire() try: # save time stamps if "job_date" in dir(self) and self.job_date is not None: - accu_group._v_attrs.time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.job_date.year, + attrs.time="%04d%02d%02d %02d:%02d:%02d.%03d"%(self.job_date.year, self.job_date.month, self.job_date.day, self.job_date.hour, @@ -393,15 +395,17 @@ class ADC_Result(MeasurementData, Resultable, Drawable, DamarisFFT, Signalpath): if self.description is not None: for (key,value) in self.description.items(): - accu_group._v_attrs.__setattr__("description_"+key,str(value)) - accu_group._v_attrs.__setattr__("sampling_rate",self.sampling_rate) + attrs.__setattr__("description_"+key,str(value)) + attrs.__setattr__("sampling_rate",self.sampling_rate) # save interval information filter=None if complib is not None: if complevel is None: complevel=9 - filter=tables.Filters(complevel=complevel,complib=complib,shuffle=1) + filter=tables.Filters(complevel=complevel, + complib=cast(_CompressionLibrary, complib), + shuffle=True) index_table=hdffile.create_table(where=accu_group, name="indices", @@ -414,7 +418,7 @@ class ADC_Result(MeasurementData, Resultable, Drawable, DamarisFFT, Signalpath): expectedrows=len(self.index)) index_table.flavor="numpy" # save channel data - new_row=index_table.row + new_row = cast(Any, index_table.row) for i in range(len(self.index)): new_row["start"]=self.index[i][0] new_row["dwelltime"]=1.0/self.sampling_rate @@ -436,27 +440,14 @@ class ADC_Result(MeasurementData, Resultable, Drawable, DamarisFFT, Signalpath): # save data time_slice_data=None if filter is not None: - chunkshape = numpy.shape(timedata) - if len(chunkshape) <= 1: - chunkshape = (min(chunkshape[0],1024*8),) - else: - chunkshape = (min(chunkshape[0],1024*8), chunkshape[1]) - if tables.__version__[0]=="1": - time_slice_data=hdffile.create_carray(accu_group, - name="adc_data", - shape=timedata.shape, - atom=tables.Int16Atom(shape=chunkshape, - flavor="numpy"), - filters=filter, - title="adc data") - else: - time_slice_data=hdffile.create_carray(accu_group, - name="adc_data", - shape=timedata.shape, - chunkshape=chunkshape, - atom=tables.Int16Atom(), - filters=filter, - title="adc data") + chunkshape = (min(timedata.shape[0], 1024 * 8), timedata.shape[1]) + time_slice_data=hdffile.create_carray(accu_group, + name="adc_data", + shape=timedata.shape, + chunkshape=chunkshape, + atom=tables.Int16Atom(), + filters=filter, + title="adc data") time_slice_data[:]=timedata else: time_slice_data=hdffile.create_array(accu_group, @@ -684,7 +675,8 @@ def read_from_hdf(hdf_node: object) -> ADC_Result | None: if not isinstance(hdf_node, tables.Group): return None - if hdf_node._v_attrs.damaris_type!="ADC_Result": + attrs = cast(Any, hdf_node._v_attrs) + if attrs.damaris_type!="ADC_Result": return None if not (hdf_node.__contains__("indices") and hdf_node.__contains__("adc_data")): @@ -696,12 +688,12 @@ def read_from_hdf(hdf_node: object) -> ADC_Result | None: adc.set_title(hdf_node._v_title) # populate description dictionary adc.description={} - for attrname in hdf_node._v_attrs._v_attrnamesuser: + for attrname in attrs._v_attrnamesuser: if attrname.startswith("description_"): - adc.description[attrname[12:]]=hdf_node._v_attrs.__getattr__(attrname) + adc.description[attrname[12:]]=attrs.__getattr__(attrname) - if "time" in dir(hdf_node._v_attrs): - timestring=hdf_node._v_attrs.__getattr__("time") + if "time" in dir(attrs): + timestring=attrs.__getattr__("time") adc.job_date=datetime.datetime(int(timestring[:4]), # year int(timestring[4:6]), # month int(timestring[6:8]), # day diff --git a/src/damaris/data/Accumulation.py b/src/damaris/data/Accumulation.py index acf8534..abe509b 100644 --- a/src/damaris/data/Accumulation.py +++ b/src/damaris/data/Accumulation.py @@ -26,12 +26,13 @@ import datetime # added by Oleg Petrov import ctypes # added by Oleg Petrov import struct # added by Oleg Petrov import os # added by Oleg Petrov -from typing import IO +from typing import IO, Any, Literal, cast _XArray = numpy.ndarray | list _YChannels = list[numpy.ndarray] _IndexList = list[tuple[int, int]] _Description = dict[str, str] +_CompressionLibrary = Literal["blosc", "blosc2", "bzip2", "lzo", "zlib"] class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath): @@ -307,7 +308,7 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) for i in range(len(xdata)): dest.write("%e"%xdata[i]) for j in range(ch_no): - if self.use_error: + if self.use_error and yerr is not None: dest.write("%s%e%s%e"%(delimiter, ydata[j][i], delimiter, yerr[j][i])) else: dest.write("%s%e"%(delimiter,ydata[j][i])) @@ -358,7 +359,7 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) self.lock.release() # ------------- added by Oleg Petrov, 10 Sep 2013 ----------------------- - def write_to_tecmag(self, destination = sys.stdout, nrecords: int = 1,\ + def write_to_tecmag(self, destination: str | IO[bytes] = sys.stdout.buffer, nrecords: int = 1,\ frequency: float = 100.,\ last_delay: float = 1.,\ receiver_phase: float = 0.,\ @@ -388,8 +389,8 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) else: record = (self.job_id/self.n)%nrecords + 1 - the_destination=destination - if type(destination) in (str,): + the_destination: IO[bytes] | None = None + if isinstance(destination, str): if record == 1 and os.path.exists(destination): os.rename(destination, os.path.dirname(destination)+'/~'+os.path.basename(destination)) @@ -420,7 +421,10 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) data = data.flatten() if record == 1: - the_destination=open(destination, "wb") + if isinstance(destination, str): + the_destination = open(destination, "wb") + else: + the_destination = destination # allocate space for all records in advance: buff = ctypes.create_string_buffer(1056+data_offset+2068) @@ -481,20 +485,24 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) struct.pack_into('4s', buff, 1056+data_offset+2060, 'PSEQ') # 'PSEQ' tag 658476 struct.pack_into('?', buff, 1056+data_offset+2064, False) # BOOLean 658480 - the_destination.write(buff) + the_destination.write(buff.raw) else: - the_destination=open(destination, "rb+") + if isinstance(destination, str): + the_destination = open(destination, "rb+") + else: + the_destination = destination buff = ctypes.create_string_buffer(4*2*npts[0]) struct.pack_into('%sf' % (2*npts[0]), buff, 0, *data) the_destination.seek(1056+4*2*npts[0]*(record-1)) - the_destination.write(buff) + the_destination.write(buff.raw) - the_destination = None ydata=None finally: + if isinstance(destination, str) and the_destination is not None: + the_destination.close() self.lock.release() # ----------------------------------------------------------------------- @@ -518,25 +526,28 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) :param complevel: compression level. """ accu_group=hdffile.create_group(where=where,name=name,title=title) - accu_group._v_attrs.damaris_type="Accumulation" + attrs = cast(Any, accu_group._v_attrs) + attrs.damaris_type="Accumulation" if self.contains_data(): self.lock.acquire() try: # save time stamps if self.time_period is not None and len(self.time_period)>0: - accu_group._v_attrs.earliest_time = self.time_period[0].isoformat(sep=' ', timespec='milliseconds') - accu_group._v_attrs.oldest_time = self.time_period[1].isoformat(sep=' ', timespec='milliseconds') + attrs.earliest_time = self.time_period[0].isoformat(sep=' ', timespec='milliseconds') + attrs.oldest_time = self.time_period[1].isoformat(sep=' ', timespec='milliseconds') if self.common_descriptions is not None: for (key,value) in self.common_descriptions.items(): - accu_group._v_attrs.__setattr__("description_"+key,str(value)) - accu_group._v_attrs.__setattr__("sampling_rate",self.sampling_rate) + attrs.__setattr__("description_"+key,str(value)) + attrs.__setattr__("sampling_rate",self.sampling_rate) filter=None if complib is not None: if complevel is None: complevel=3 - filter=tables.Filters(complevel=complevel,complib=complib,shuffle=1) + filter=tables.Filters(complevel=complevel, + complib=cast(_CompressionLibrary, complib), + shuffle=True) # save interval information # start save index_table @@ -553,7 +564,7 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) expectedrows=len(self.index)) index_table.flavor="numpy" # save interval data - new_row=index_table.row + new_row = cast(Any, index_table.row) for i in range(len(self.index)): new_row["start"]=self.index[i][0] new_row["dwelltime"]=1.0/self.sampling_rate @@ -574,7 +585,7 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) title="job_ids used for accumulation", filters=filter, expectedrows=len(self.job_ids)) - new_row=jobid_table.row + new_row = cast(Any, jobid_table.row) for i,job_id in enumerate(self.job_ids): new_row["job_date"] = self.job_ids[job_id].isoformat(timespec='milliseconds') new_row["job_id"]=job_id @@ -595,27 +606,14 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) # save data time_slice_data=None if filter is not None: - chunkshape=timedata.shape - if len(chunkshape) <= 1: - chunkshape = (min(chunkshape[0],1024*8),) - else: - chunkshape = (min(chunkshape[0],1024*8), chunkshape[1]) - if tables.__version__[0]=="1": - time_slice_data=hdffile.create_carray(accu_group, - name="accu_data", - shape=timedata.shape, - atom=tables.Float32Atom(shape=chunkshape, - flavor="numpy"), - filters=filter, - title="accu data") - else: - time_slice_data=hdffile.create_carray(accu_group, - name="accu_data", - shape=timedata.shape, - chunkshape=chunkshape, - atom=tables.Float32Atom(), - filters=filter, - title="accu data") + chunkshape = (min(timedata.shape[0], 1024 * 8), timedata.shape[1]) + time_slice_data=hdffile.create_carray(accu_group, + name="accu_data", + shape=timedata.shape, + chunkshape=chunkshape, + atom=tables.Float32Atom(), + filters=filter, + title="accu data") time_slice_data[:]=timedata else: @@ -1111,7 +1109,8 @@ class Accumulation(MeasurementData, Errorable, Drawable, DamarisFFT, Signalpath) self.set_title(self.__title_pattern % self.n) if hasattr(other, "is_clipped"): self.is_clipped = other.is_clipped - self.common_descriptions=other.common_descriptions.copy() + self.common_descriptions = (other.common_descriptions.copy() + if other.common_descriptions is not None else None) self.time_period=other.time_period[:] if other.time_period is not None else None self.job_id = other.job_id # added by Oleg Petrov self.job_ids.update(other.job_ids) @@ -1213,7 +1212,8 @@ def read_from_hdf(hdf_node: object) -> "Accumulation | None": if not isinstance(hdf_node, tables.Group): return None - if hdf_node._v_attrs.damaris_type!="Accumulation": + attrs = cast(Any, hdf_node._v_attrs) + if attrs.damaris_type!="Accumulation": return None if not (hdf_node.__contains__("indices") and hdf_node.__contains__("accu_data")): @@ -1224,13 +1224,13 @@ def read_from_hdf(hdf_node: object) -> "Accumulation | None": # populate description dictionary accu.common_descriptions={} - for attrname in hdf_node._v_attrs._v_attrnamesuser: + for attrname in attrs._v_attrnamesuser: if attrname.startswith("description_"): - accu.common_descriptions[attrname[12:]]=hdf_node._v_attrs.__getattr__(attrname) + accu.common_descriptions[attrname[12:]]=attrs.__getattr__(attrname) earliest_time=None - if "earliest_time" in dir(hdf_node._v_attrs): - timestring=hdf_node._v_attrs.__getattr__("earliest_time") + if "earliest_time" in dir(attrs): + timestring=attrs.__getattr__("earliest_time") # Handle both formats: "20260705 19:13:59.162" and "2026-07-05 19:13:59.162" if '-' in timestring: # New format: "2026-07-05 19:13:59.162" @@ -1261,8 +1261,8 @@ def read_from_hdf(hdf_node: object) -> "Accumulation | None": ) oldest_time=None - if "oldest_time" in dir(hdf_node._v_attrs): - timestring=hdf_node._v_attrs.__getattr__("oldest_time") + if "oldest_time" in dir(attrs): + timestring=attrs.__getattr__("oldest_time") # Handle both formats: "20260705 19:13:59.162" and "2026-07-05 19:13:59.162" if '-' in timestring: # New format: "2026-07-05 19:13:59.162" diff --git a/src/damaris/data/MeasurementResult.py b/src/damaris/data/MeasurementResult.py index 8f3853a..2c430d8 100644 --- a/src/damaris/data/MeasurementResult.py +++ b/src/damaris/data/MeasurementResult.py @@ -4,7 +4,7 @@ import sys import tables import numpy import collections -from typing import IO +from typing import IO, Any, Literal, cast from . import Drawable ## provide gaussian statistics for a series of measured data points @@ -31,7 +31,10 @@ class AccumulatedValue: self.y2=self.y**2 self.n=1 elif mean_err is None: - self.n=max(1, int(n)) + if n is None: + self.n=1 + else: + self.n=max(1, n) self.y=float(mean)*self.n self.y2=(float(mean)**2)*self.n elif n is None: @@ -240,7 +243,9 @@ class MeasurementResult(Drawable.Drawable, collections.UserDict): if complib is not None: if complevel is None: complevel=9 - filter=tables.Filters(complevel=complevel,complib=complib,shuffle=1) + filter=tables.Filters(complevel=complevel, + complib=cast(Literal["blosc", "blosc2", "bzip2", "lzo", "zlib"], complib), + shuffle=True) mr_table=hdffile.create_table(where=where,name=name, description=h5_table_format, @@ -248,11 +253,12 @@ class MeasurementResult(Drawable.Drawable, collections.UserDict): filters=filter, expectedrows=len(self)) mr_table.flavor="numpy" - mr_table.attrs.damaris_type="MeasurementResult" + attrs = cast(Any, mr_table.attrs) + attrs.damaris_type="MeasurementResult" self.lock.acquire() try: - mr_table.attrs.quantity_name=self.quantity_name - row=mr_table.row + attrs.quantity_name=self.quantity_name + row = cast(Any, mr_table.row) xdata=self.get_xdata() if xdata.shape[0]!=0: for x in xdata: @@ -276,10 +282,11 @@ def read_from_hdf(hdf_node: object) -> "MeasurementResult | None": if not isinstance(hdf_node, tables.Table): return None - if hdf_node._v_attrs.damaris_type!="MeasurementResult": + attrs = cast(Any, hdf_node._v_attrs) + if attrs.damaris_type!="MeasurementResult": return None - mr=MeasurementResult(hdf_node._v_attrs.quantity_name) + mr=MeasurementResult(attrs.quantity_name) for r in hdf_node.iterrows(): mr[r["x"]]=AccumulatedValue(r["y"],r["y_err"],r["n"]) diff --git a/tests/test_Accumulation.py b/tests/test_Accumulation.py index d5c3172..9e87980 100644 --- a/tests/test_Accumulation.py +++ b/tests/test_Accumulation.py @@ -360,6 +360,20 @@ class TestAccumulation(unittest.TestCase): r *= 2 np.testing.assert_array_equal(accu.y[0], np.array([10, 20, 30], dtype="float32")) + def test_empty_iadd_accumulation_without_common_descriptions(self): + """An accumulation without common descriptions can be copied in-place.""" + other = Accumulation(x=np.array([0.0, 1.0]), + y=[np.array([10.0, 20.0])], + n=1, + index=[(0, 1)], + sampl_freq=1000.0) + accu = Accumulation() + + accu += other + + self.assertIsNone(accu.common_descriptions) + np.testing.assert_array_equal(accu.y[0], other.y[0]) + def test_accumulation_plus_accumulation_keeps_both_job_ids(self): """accu1 + accu2 must keep the provenance of both operands.""" adc1 = self.create_adc_result()