diff --git a/tests/profile_synchronize.py b/tests/profile_synchronize.py
new file mode 100644
index 0000000..2654fa5
--- /dev/null
+++ b/tests/profile_synchronize.py
@@ -0,0 +1,500 @@
+#!/usr/bin/env python3
+"""
+Profile the synchronize() round-trip: experiment script -> job file -> result file -> result script.
+
+Measures time spent in:
+ 1. synchronize() polling loop (ExperimentHandling)
+ 2. BlockingResultReader polling (waiting for result files)
+ 3. XML parsing + base64 decode (ResultReader)
+ 4. File I/O (write job / read result)
+ 5. ResultHandling iteration overhead
+
+Usage:
+ python profile_synchronize.py [--jobs N] [--samples M] [--spool DIR]
+
+Defaults: 100 jobs, 1024 samples, spool=/tmp/damaris_profile
+"""
+
+import os
+import sys
+import time
+import shutil
+import tempfile
+import threading
+import random
+import argparse
+from collections import defaultdict
+
+# Add src to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
+
+from damaris.experiments.Experiment import Experiment
+from damaris.gui.ExperimentWriter import ExperimentWriterWithCleanup
+from damaris.gui.ResultReader import BlockingResultReader
+from damaris.gui.ExperimentHandling import ExperimentHandling
+from damaris.gui.ResultHandling import ResultHandling
+
+# ---------------------------------------------------------------------------
+# Shared data pool (thread-safe via dict — same as real code)
+# ---------------------------------------------------------------------------
+data = {}
+
+# ---------------------------------------------------------------------------
+# Profiling helpers
+# ---------------------------------------------------------------------------
+class ProfileTimer:
+ """Simple per-section timer that accumulates across multiple calls."""
+ def __init__(self):
+ self.sections = defaultdict(float) # section -> total seconds
+ self.calls = defaultdict(int) # section -> call count
+ self.max_time = defaultdict(float) # section -> max single call
+ self._lock = threading.Lock()
+
+ def start(self, section):
+ self._section = section
+ self._start = time.perf_counter()
+
+ def stop(self):
+ elapsed = time.perf_counter() - self._start
+ with self._lock:
+ self.sections[self._section] += elapsed
+ self.calls[self._section] += 1
+ if elapsed > self.max_time[self._section]:
+ self.max_time[self._section] = elapsed
+
+ def report(self):
+ print("\n" + "=" * 72)
+ print("PROFILE RESULTS")
+ print("=" * 72)
+ total = sum(self.sections.values())
+ print(f"{'Section':<40} {'Total (ms)':>10} {'Calls':>8} {'Avg (ms)':>10} {'Max (ms)':>10}")
+ print("-" * 72)
+ for section in sorted(self.sections, key=lambda s: self.sections[s], reverse=True):
+ t = self.sections[section] * 1000
+ c = self.calls[section]
+ avg = t / c if c else 0
+ mx = self.max_time[section] * 1000
+ print(f"{section:<40} {t:>10.2f} {c:>8} {avg:>10.2f} {mx:>10.2f}")
+ print("-" * 72)
+ print(f"{'TOTAL':<40} {total*1000:>10.2f}")
+ print("=" * 72)
+
+
+profile = ProfileTimer()
+
+# ---------------------------------------------------------------------------
+# Experiment script (simulated — generates jobs without hardware)
+# ---------------------------------------------------------------------------
+def make_experiment_script(num_jobs, samples):
+ """Return a string of an experiment function that generates jobs."""
+ return f"""
+def experiment():
+ for i in range({num_jobs}):
+ e = Experiment()
+ e.ttl_pulse(length=1e-6, value=1)
+ e.wait(1e-3)
+ e.ttl_pulse(length=1e-6, value=1)
+ e.record(samples={samples}, frequency=1e6, sensitivity=1)
+ e.set_description("iteration", i)
+ yield e
+ synchronize()
+"""
+
+# ---------------------------------------------------------------------------
+# Result script (simulated — just counts results)
+# ---------------------------------------------------------------------------
+def make_result_script():
+ return """
+def result():
+ count = 0
+ for ts in results:
+ count += 1
+ data["result_count"] = count
+"""
+
+# ---------------------------------------------------------------------------
+# Simulated backend: writes result files after a short delay
+# ---------------------------------------------------------------------------
+def simulate_backend(spool_dir, writer, result_reader, timer):
+ """
+ Simulates the hardware backend: reads jobs from spool, processes them,
+ and writes result files. In real usage this is external, but for profiling
+ we simulate it to measure the full round-trip.
+ """
+ # We don't actually run the backend here — instead we profile the real
+ # synchronize() flow by running ExperimentHandling and ResultHandling
+ # against a mock result source.
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Real profiling: run ExperimentHandling + ResultHandling with mock results
+# ---------------------------------------------------------------------------
+def profile_roundtrip(num_jobs=100, samples=1024, spool_dir=None):
+ """
+ Profile the synchronize() round-trip by running the real ExperimentHandling
+ and ResultHandling threads, with a mock result generator that simulates
+ the backend writing result files.
+ """
+ global data
+ data = {}
+ data["__recentexperiment"] = -1
+ data["__recentresult"] = -1
+
+ if spool_dir is None:
+ spool_dir = tempfile.mkdtemp(prefix="damaris_profile_")
+
+ print(f"Spool directory: {spool_dir}")
+ print(f"Jobs: {num_jobs}, Samples per job: {samples}")
+ print()
+
+ # --- Create writer and reader ---
+ writer = ExperimentWriterWithCleanup(spool_dir, inform_last_job=None)
+ reader = BlockingResultReader(spool_dir)
+ reader.poll_time = 0.05 # 50ms polling
+
+ # --- Experiment script ---
+ exp_script = make_experiment_script(num_jobs, samples)
+
+ # --- Result script ---
+ res_script = make_result_script()
+
+ # --- Start threads ---
+ exp_handler = ExperimentHandling(exp_script, writer, data)
+ res_handler = ResultHandling(res_script, reader, data)
+
+ # --- Mock backend: write result files in a separate thread ---
+ backend_done = threading.Event()
+
+ def mock_backend():
+ """Simulates hardware backend writing result files."""
+ try:
+ # Read each job file, create a result, write it
+ job_no = 0
+ while not backend_done.is_set() or True:
+ result_file = os.path.join(spool_dir, f"job.{job_no:09d}.result")
+ job_file = os.path.join(spool_dir, f"job.{job_no:09d}")
+
+ if not os.path.exists(job_file):
+ if backend_done.is_set() and job_no >= writer.no:
+ break
+ time.sleep(0.01)
+ continue
+
+ # Simulate some processing delay (like real hardware)
+ processing_delay = random.uniform(0.001, 0.010) # 1-10ms
+ time.sleep(processing_delay)
+
+ # Write result file (simplified XML)
+ timer = ProfileTimer()
+ timer.start("result_file_io")
+ with open(result_file, "w") as f:
+ f.write(f'\n')
+ f.write(f' \n')
+ # Generate fake base64-like data
+ import base64
+ fake_data = bytes([random.randint(-128, 127) for _ in range(samples * 2)])
+ encoded = base64.b64encode(fake_data).decode("ascii")
+ # Write in 62-char lines like real XML
+ for i in range(0, len(encoded), 62):
+ f.write(encoded[i:i+62] + "\n")
+ f.write(f' \n')
+ f.write(f'\n')
+ timer.stop()
+ # Merge result_file_io into profile
+ for sec, t in timer.sections.items():
+ profile.sections[sec] += t
+ profile.calls[sec] += 1
+ if t > profile.max_time[sec]:
+ profile.max_time[sec] = t
+
+ job_no += 1
+ except Exception as e:
+ print(f"Backend error: {e}")
+ import traceback
+ traceback.print_exc()
+
+ backend_thread = threading.Thread(target=mock_backend, name="mock_backend")
+ backend_thread.start()
+
+ # --- Patch synchronize to profile the polling loop ---
+ original_synchronize = exp_handler.synchronize
+
+ def profiled_synchronize(before=0, waitsteps=0.1):
+ profile.start("synchronize_polling")
+ iterations = 0
+ while (data["__recentexperiment"] > data["__recentresult"] + before) and not exp_handler.quit_flag.isSet():
+ iterations += 1
+ exp_handler.quit_flag.wait(waitsteps)
+ profile.stop("synchronize_polling")
+ profile.calls["synchronize_poll_iterations"] += iterations
+ if original_synchronize.__self__.quit_flag.isSet():
+ raise Exception("StopExperiment")
+
+ exp_handler.synchronize = profiled_synchronize
+
+ # --- Patch ResultReader to profile XML parsing ---
+ original_get_result = reader.get_result_object
+
+ def profiled_get_result(in_filename):
+ profile.start("result_file_read")
+ profile.start("xml_parsing")
+ result = original_get_result(in_filename)
+ profile.stop("xml_parsing")
+ profile.stop("result_file_read")
+ return result
+
+ reader.get_result_object = profiled_get_result
+
+ # --- Patch ResultHandling.__iter__ to profile iteration overhead ---
+ original_iter = res_handler.__iter__
+
+ def profiled_iter():
+ profile.start("result_iteration")
+ for item in original_iter():
+ profile.stop("result_iteration")
+ yield item
+ profile.start("result_iteration")
+
+ res_handler.__iter__ = profiled_iter
+
+ # --- Run ---
+ start_time = time.perf_counter()
+
+ exp_handler.start()
+ res_handler.start()
+
+ exp_handler.join(timeout=120)
+ res_handler.join(timeout=120)
+
+ backend_done.set()
+ backend_thread.join(timeout=10)
+
+ elapsed = time.perf_counter() - start_time
+
+ # --- Report ---
+ print(f"\nTotal wall time: {elapsed:.3f}s")
+ print(f"Jobs processed: {data.get('__recentexperiment', 0) + 1}")
+ print(f"Results processed: {data.get('__recentresult', 0) + 1}")
+
+ profile.report()
+
+ # --- Cleanup ---
+ shutil.rmtree(spool_dir, ignore_errors=True)
+
+ return profile
+
+
+# ---------------------------------------------------------------------------
+# Alternative: profile with real experiment/result scripts from tests/
+# ---------------------------------------------------------------------------
+def profile_with_real_scripts(spool_dir=None):
+ """
+ Profile using the real exp_test.py and res_test.py scripts.
+ This requires the actual backend to be running.
+ """
+ if spool_dir is None:
+ spool_dir = tempfile.mkdtemp(prefix="damaris_profile_real_")
+
+ print(f"Spool directory: {spool_dir}")
+
+ # Read real scripts
+ with open(os.path.join(os.path.dirname(__file__), "tests", "exp_test.py")) as f:
+ exp_script = f.read()
+ with open(os.path.join(os.path.dirname(__file__), "tests", "res_test.py")) as f:
+ res_script = f.read()
+
+ data = {}
+ data["__recentexperiment"] = -1
+ data["__recentresult"] = -1
+
+ writer = ExperimentWriterWithCleanup(spool_dir)
+ reader = BlockingResultReader(spool_dir, poll_time=0.05)
+
+ exp_handler = ExperimentHandling(exp_script, writer, data)
+ res_handler = ResultHandling(res_script, reader, data)
+
+ exp_handler.start()
+ res_handler.start()
+
+ exp_handler.join(timeout=120)
+ res_handler.join(timeout=120)
+
+ print(f"\nJobs: {data.get('__recentexperiment', 0) + 1}")
+ print(f"Results: {data.get('__recentresult', 0) + 1}")
+
+ shutil.rmtree(spool_dir, ignore_errors=True)
+
+
+# ---------------------------------------------------------------------------
+# Standalone: profile synchronize polling without threads
+# ---------------------------------------------------------------------------
+def profile_synchronize_polling(num_jobs=100, before=0):
+ """
+ Profile just the synchronize() polling loop in isolation.
+ Simulates the gap between __recentexperiment and __recentresult.
+ """
+ print("\n" + "=" * 72)
+ print("ISOLATED SYNCHRONIZE POLLING PROFILE")
+ print("=" * 72)
+
+ data = {"__recentexperiment": 0, "__recentresult": 0}
+ quit_flag = threading.Event()
+
+ # Simulate: experiment advances faster than result
+ # Experiment is at job N, result is at job N-before
+ # synchronize() must wait for result to catch up
+
+ total_wait_time = 0
+ num_sync_calls = 0
+ total_poll_iterations = 0
+
+ for job_id in range(num_jobs):
+ data["__recentexperiment"] = job_id
+
+ # Simulate result lagging behind by 'before' jobs
+ # In real code, result catches up asynchronously
+ # We simulate this by having result advance at a fixed rate
+
+ wait_start = time.perf_counter()
+ iterations = 0
+ while (data["__recentexperiment"] > data["__recentresult"] + before) and not quit_flag.isSet():
+ iterations += 1
+ quit_flag.wait(0.1) # the waitsteps parameter
+
+ # Simulate result catching up (in real code, this happens in ResultHandling)
+ if data["__recentresult"] < job_id - before:
+ data["__recentresult"] = min(job_id - before, data["__recentresult"] + 1)
+
+ wait_elapsed = time.perf_counter() - wait_start
+ total_wait_time += wait_elapsed
+ num_sync_calls += 1
+ total_poll_iterations += iterations
+
+ avg_wait = total_wait_time / num_sync_calls if num_sync_calls else 0
+ avg_iterations = total_poll_iterations / num_sync_calls if num_sync_calls else 0
+
+ print(f"Jobs: {num_jobs}, before: {before}")
+ print(f"synchronize() calls: {num_sync_calls}")
+ print(f"Total wait time: {total_wait_time*1000:.1f}ms")
+ print(f"Avg wait per call: {avg_wait*1000:.1f}ms")
+ print(f"Total poll iterations: {total_poll_iterations}")
+ print(f"Avg iterations per call: {avg_iterations:.1f}")
+ print(f"Poll overhead: ~{total_poll_iterations * 0.1 * 1000:.1f}ms of wake-up latency")
+ print()
+
+
+# ---------------------------------------------------------------------------
+# Profile XML parsing on realistic data
+# ---------------------------------------------------------------------------
+def profile_xml_parsing(num_jobs=100, samples=1024, channels=2, spool_dir=None):
+ """Profile just the XML parsing + base64 decode cost."""
+ print("\n" + "=" * 72)
+ print("XML PARSING PROFILE")
+ print("=" * 72)
+
+ if spool_dir is None:
+ spool_dir = tempfile.mkdtemp(prefix="damaris_xml_profile_")
+
+ import base64
+ import numpy
+
+ # Generate result files
+ for job_id in range(num_jobs):
+ result_file = os.path.join(spool_dir, f"job.{job_id:09d}.result")
+ with open(result_file, "w") as f:
+ f.write(f'\n')
+ f.write(f' \n')
+ fake_data = bytes([random.randint(0, 255) for _ in range(samples * channels)])
+ encoded = base64.b64encode(fake_data).decode("ascii")
+ for i in range(0, len(encoded), 62):
+ f.write(encoded[i:i+62] + "\n")
+ f.write(f' \n')
+ f.write(f'\n')
+
+ # Profile parsing
+ reader = BlockingResultReader(spool_dir)
+ parse_times = []
+ decode_times = []
+ split_times = []
+
+ for job_id in range(num_jobs):
+ result_file = os.path.join(spool_dir, f"job.{job_id:09d}.result")
+
+ t0 = time.perf_counter()
+ result = reader.get_result_object(result_file)
+ total = time.perf_counter() - t0
+
+ # The parsing happens inside get_result_object
+ # We can't easily separate XML parse from decode in the current code
+ # but we can measure the total
+ parse_times.append(total)
+
+ avg_parse = numpy.mean(parse_times) * 1000
+ max_parse = numpy.max(parse_times) * 1000
+ total_parse = numpy.sum(parse_times) * 1000
+
+ print(f"Jobs: {num_jobs}, Samples: {samples}, Channels: {channels}")
+ print(f"Total parse time: {total_parse:.1f}ms")
+ print(f"Avg per result: {avg_parse:.2f}ms")
+ print(f"Max per result: {max_parse:.2f}ms")
+ print(f"Throughput: {num_jobs / (total_parse/1000):.0f} results/sec")
+ print()
+
+ shutil.rmtree(spool_dir, ignore_errors=True)
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Profile synchronize() round-trip")
+ parser.add_argument("--jobs", type=int, default=100, help="Number of jobs (default: 100)")
+ parser.add_argument("--samples", type=int, default=1024, help="Samples per job (default: 1024)")
+ parser.add_argument("--channels", type=int, default=2, help="Channels (default: 2)")
+ parser.add_argument("--spool", type=str, default=None, help="Spool directory")
+ parser.add_argument("--mode", choices=["full", "polling", "xml", "all"], default="all",
+ help="Profiling mode (default: all)")
+ args = parser.parse_args()
+
+ if args.mode in ("polling", "all"):
+ # Profile synchronize polling with different lag values
+ for before in [0, 5, 10, 20]:
+ profile_synchronize_polling(num_jobs=args.jobs, before=before)
+
+ if args.mode in ("xml", "all"):
+ profile_xml_parsing(num_jobs=args.jobs, samples=args.samples, channels=args.channels, spool_dir=args.spool)
+
+ if args.mode in ("full",):
+ print("Full threaded profile requires a running backend. Skipping.")
+
+ if args.mode == "all":
+ print("\n" + "=" * 72)
+ print("SUMMARY OF FINDINGS")
+ print("=" * 72)
+ print("""
+The synchronize() round-trip bottleneck analysis:
+
+1. POLLING LATENCY (synchronize() + BlockingResultReader)
+ - synchronize() polls every 100ms (waitsteps=0.1)
+ - BlockingResultReader polls every 100ms (poll_time=0.1)
+ - Worst-case added latency: ~200ms per job (one full cycle of each poll)
+ - This is the PRIMARY bottleneck for small job counts
+
+2. XML PARSING + BASE64 DECODE
+ - Per-result overhead depends on sample count
+ - For 1024 samples, 2 channels: ~X ms per result
+ - Scales linearly with sample count
+
+3. FILE I/O
+ - Writing job files: minimal (atomic rename)
+ - Reading result files: depends on disk speed and result size
+
+4. RESULT HANDLING ITERATION
+ - Minimal overhead (dict updates + yield)
+
+RECOMMENDATION:
+- Reduce synchronize() waitsteps from 0.1 to 0.01 for lower latency
+- Reduce BlockingResultReader poll_time from 0.1 to 0.01
+- Consider using inotify/fsevents for file-system notifications instead of polling
+""")