Public Access
501 lines
19 KiB
Python
501 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Profile the synchronize() round-trip: experiment script -> job file -> result file -> result script.
|
|
|
|
Measures time spent in:
|
|
1. synchronize() polling loop (ExperimentHandling)
|
|
2. BlockingResultReader polling (waiting for result files)
|
|
3. XML parsing + base64 decode (ResultReader)
|
|
4. File I/O (write job / read result)
|
|
5. ResultHandling iteration overhead
|
|
|
|
Usage:
|
|
python profile_synchronize.py [--jobs N] [--samples M] [--spool DIR]
|
|
|
|
Defaults: 100 jobs, 1024 samples, spool=/tmp/damaris_profile
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import shutil
|
|
import tempfile
|
|
import threading
|
|
import random
|
|
import argparse
|
|
from collections import defaultdict
|
|
|
|
# Add src to path
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "src"))
|
|
|
|
from damaris.experiments.Experiment import Experiment
|
|
from damaris.gui.ExperimentWriter import ExperimentWriterWithCleanup
|
|
from damaris.gui.ResultReader import BlockingResultReader
|
|
from damaris.gui.ExperimentHandling import ExperimentHandling
|
|
from damaris.gui.ResultHandling import ResultHandling
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Shared data pool (thread-safe via dict — same as real code)
|
|
# ---------------------------------------------------------------------------
|
|
data = {}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Profiling helpers
|
|
# ---------------------------------------------------------------------------
|
|
class ProfileTimer:
|
|
"""Simple per-section timer that accumulates across multiple calls."""
|
|
def __init__(self):
|
|
self.sections = defaultdict(float) # section -> total seconds
|
|
self.calls = defaultdict(int) # section -> call count
|
|
self.max_time = defaultdict(float) # section -> max single call
|
|
self._lock = threading.Lock()
|
|
|
|
def start(self, section):
|
|
self._section = section
|
|
self._start = time.perf_counter()
|
|
|
|
def stop(self):
|
|
elapsed = time.perf_counter() - self._start
|
|
with self._lock:
|
|
self.sections[self._section] += elapsed
|
|
self.calls[self._section] += 1
|
|
if elapsed > self.max_time[self._section]:
|
|
self.max_time[self._section] = elapsed
|
|
|
|
def report(self):
|
|
print("\n" + "=" * 72)
|
|
print("PROFILE RESULTS")
|
|
print("=" * 72)
|
|
total = sum(self.sections.values())
|
|
print(f"{'Section':<40} {'Total (ms)':>10} {'Calls':>8} {'Avg (ms)':>10} {'Max (ms)':>10}")
|
|
print("-" * 72)
|
|
for section in sorted(self.sections, key=lambda s: self.sections[s], reverse=True):
|
|
t = self.sections[section] * 1000
|
|
c = self.calls[section]
|
|
avg = t / c if c else 0
|
|
mx = self.max_time[section] * 1000
|
|
print(f"{section:<40} {t:>10.2f} {c:>8} {avg:>10.2f} {mx:>10.2f}")
|
|
print("-" * 72)
|
|
print(f"{'TOTAL':<40} {total*1000:>10.2f}")
|
|
print("=" * 72)
|
|
|
|
|
|
profile = ProfileTimer()
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Experiment script (simulated — generates jobs without hardware)
|
|
# ---------------------------------------------------------------------------
|
|
def make_experiment_script(num_jobs, samples):
|
|
"""Return a string of an experiment function that generates jobs."""
|
|
return f"""
|
|
def experiment():
|
|
for i in range({num_jobs}):
|
|
e = Experiment()
|
|
e.ttl_pulse(length=1e-6, value=1)
|
|
e.wait(1e-3)
|
|
e.ttl_pulse(length=1e-6, value=1)
|
|
e.record(samples={samples}, frequency=1e6, sensitivity=1)
|
|
e.set_description("iteration", i)
|
|
yield e
|
|
synchronize()
|
|
"""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Result script (simulated — just counts results)
|
|
# ---------------------------------------------------------------------------
|
|
def make_result_script():
|
|
return """
|
|
def result():
|
|
count = 0
|
|
for ts in results:
|
|
count += 1
|
|
data["result_count"] = count
|
|
"""
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Simulated backend: writes result files after a short delay
|
|
# ---------------------------------------------------------------------------
|
|
def simulate_backend(spool_dir, writer, result_reader, timer):
|
|
"""
|
|
Simulates the hardware backend: reads jobs from spool, processes them,
|
|
and writes result files. In real usage this is external, but for profiling
|
|
we simulate it to measure the full round-trip.
|
|
"""
|
|
# We don't actually run the backend here — instead we profile the real
|
|
# synchronize() flow by running ExperimentHandling and ResultHandling
|
|
# against a mock result source.
|
|
pass
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Real profiling: run ExperimentHandling + ResultHandling with mock results
|
|
# ---------------------------------------------------------------------------
|
|
def profile_roundtrip(num_jobs=100, samples=1024, spool_dir=None):
|
|
"""
|
|
Profile the synchronize() round-trip by running the real ExperimentHandling
|
|
and ResultHandling threads, with a mock result generator that simulates
|
|
the backend writing result files.
|
|
"""
|
|
global data
|
|
data = {}
|
|
data["__recentexperiment"] = -1
|
|
data["__recentresult"] = -1
|
|
|
|
if spool_dir is None:
|
|
spool_dir = tempfile.mkdtemp(prefix="damaris_profile_")
|
|
|
|
print(f"Spool directory: {spool_dir}")
|
|
print(f"Jobs: {num_jobs}, Samples per job: {samples}")
|
|
print()
|
|
|
|
# --- Create writer and reader ---
|
|
writer = ExperimentWriterWithCleanup(spool_dir, inform_last_job=None)
|
|
reader = BlockingResultReader(spool_dir)
|
|
reader.poll_time = 0.05 # 50ms polling
|
|
|
|
# --- Experiment script ---
|
|
exp_script = make_experiment_script(num_jobs, samples)
|
|
|
|
# --- Result script ---
|
|
res_script = make_result_script()
|
|
|
|
# --- Start threads ---
|
|
exp_handler = ExperimentHandling(exp_script, writer, data)
|
|
res_handler = ResultHandling(res_script, reader, data)
|
|
|
|
# --- Mock backend: write result files in a separate thread ---
|
|
backend_done = threading.Event()
|
|
|
|
def mock_backend():
|
|
"""Simulates hardware backend writing result files."""
|
|
try:
|
|
# Read each job file, create a result, write it
|
|
job_no = 0
|
|
while not backend_done.is_set() or True:
|
|
result_file = os.path.join(spool_dir, f"job.{job_no:09d}.result")
|
|
job_file = os.path.join(spool_dir, f"job.{job_no:09d}")
|
|
|
|
if not os.path.exists(job_file):
|
|
if backend_done.is_set() and job_no >= writer.no:
|
|
break
|
|
time.sleep(0.01)
|
|
continue
|
|
|
|
# Simulate some processing delay (like real hardware)
|
|
processing_delay = random.uniform(0.001, 0.010) # 1-10ms
|
|
time.sleep(processing_delay)
|
|
|
|
# Write result file (simplified XML)
|
|
timer = ProfileTimer()
|
|
timer.start("result_file_io")
|
|
with open(result_file, "w") as f:
|
|
f.write(f'<result job="{job_no}">\n')
|
|
f.write(f' <adcdata rate="1000000.0" channels="2" samples="{samples}">\n')
|
|
# Generate fake base64-like data
|
|
import base64
|
|
fake_data = bytes([random.randint(-128, 127) for _ in range(samples * 2)])
|
|
encoded = base64.b64encode(fake_data).decode("ascii")
|
|
# Write in 62-char lines like real XML
|
|
for i in range(0, len(encoded), 62):
|
|
f.write(encoded[i:i+62] + "\n")
|
|
f.write(f' </adcdata>\n')
|
|
f.write(f'</result>\n')
|
|
timer.stop()
|
|
# Merge result_file_io into profile
|
|
for sec, t in timer.sections.items():
|
|
profile.sections[sec] += t
|
|
profile.calls[sec] += 1
|
|
if t > profile.max_time[sec]:
|
|
profile.max_time[sec] = t
|
|
|
|
job_no += 1
|
|
except Exception as e:
|
|
print(f"Backend error: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
backend_thread = threading.Thread(target=mock_backend, name="mock_backend")
|
|
backend_thread.start()
|
|
|
|
# --- Patch synchronize to profile the polling loop ---
|
|
original_synchronize = exp_handler.synchronize
|
|
|
|
def profiled_synchronize(before=0, waitsteps=0.1):
|
|
profile.start("synchronize_polling")
|
|
iterations = 0
|
|
while (data["__recentexperiment"] > data["__recentresult"] + before) and not exp_handler.quit_flag.isSet():
|
|
iterations += 1
|
|
exp_handler.quit_flag.wait(waitsteps)
|
|
profile.stop("synchronize_polling")
|
|
profile.calls["synchronize_poll_iterations"] += iterations
|
|
if original_synchronize.__self__.quit_flag.isSet():
|
|
raise Exception("StopExperiment")
|
|
|
|
exp_handler.synchronize = profiled_synchronize
|
|
|
|
# --- Patch ResultReader to profile XML parsing ---
|
|
original_get_result = reader.get_result_object
|
|
|
|
def profiled_get_result(in_filename):
|
|
profile.start("result_file_read")
|
|
profile.start("xml_parsing")
|
|
result = original_get_result(in_filename)
|
|
profile.stop("xml_parsing")
|
|
profile.stop("result_file_read")
|
|
return result
|
|
|
|
reader.get_result_object = profiled_get_result
|
|
|
|
# --- Patch ResultHandling.__iter__ to profile iteration overhead ---
|
|
original_iter = res_handler.__iter__
|
|
|
|
def profiled_iter():
|
|
profile.start("result_iteration")
|
|
for item in original_iter():
|
|
profile.stop("result_iteration")
|
|
yield item
|
|
profile.start("result_iteration")
|
|
|
|
res_handler.__iter__ = profiled_iter
|
|
|
|
# --- Run ---
|
|
start_time = time.perf_counter()
|
|
|
|
exp_handler.start()
|
|
res_handler.start()
|
|
|
|
exp_handler.join(timeout=120)
|
|
res_handler.join(timeout=120)
|
|
|
|
backend_done.set()
|
|
backend_thread.join(timeout=10)
|
|
|
|
elapsed = time.perf_counter() - start_time
|
|
|
|
# --- Report ---
|
|
print(f"\nTotal wall time: {elapsed:.3f}s")
|
|
print(f"Jobs processed: {data.get('__recentexperiment', 0) + 1}")
|
|
print(f"Results processed: {data.get('__recentresult', 0) + 1}")
|
|
|
|
profile.report()
|
|
|
|
# --- Cleanup ---
|
|
shutil.rmtree(spool_dir, ignore_errors=True)
|
|
|
|
return profile
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Alternative: profile with real experiment/result scripts from tests/
|
|
# ---------------------------------------------------------------------------
|
|
def profile_with_real_scripts(spool_dir=None):
|
|
"""
|
|
Profile using the real exp_test.py and res_test.py scripts.
|
|
This requires the actual backend to be running.
|
|
"""
|
|
if spool_dir is None:
|
|
spool_dir = tempfile.mkdtemp(prefix="damaris_profile_real_")
|
|
|
|
print(f"Spool directory: {spool_dir}")
|
|
|
|
# Read real scripts
|
|
with open(os.path.join(os.path.dirname(__file__), "tests", "exp_test.py")) as f:
|
|
exp_script = f.read()
|
|
with open(os.path.join(os.path.dirname(__file__), "tests", "res_test.py")) as f:
|
|
res_script = f.read()
|
|
|
|
data = {}
|
|
data["__recentexperiment"] = -1
|
|
data["__recentresult"] = -1
|
|
|
|
writer = ExperimentWriterWithCleanup(spool_dir)
|
|
reader = BlockingResultReader(spool_dir, poll_time=0.05)
|
|
|
|
exp_handler = ExperimentHandling(exp_script, writer, data)
|
|
res_handler = ResultHandling(res_script, reader, data)
|
|
|
|
exp_handler.start()
|
|
res_handler.start()
|
|
|
|
exp_handler.join(timeout=120)
|
|
res_handler.join(timeout=120)
|
|
|
|
print(f"\nJobs: {data.get('__recentexperiment', 0) + 1}")
|
|
print(f"Results: {data.get('__recentresult', 0) + 1}")
|
|
|
|
shutil.rmtree(spool_dir, ignore_errors=True)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Standalone: profile synchronize polling without threads
|
|
# ---------------------------------------------------------------------------
|
|
def profile_synchronize_polling(num_jobs=100, before=0):
|
|
"""
|
|
Profile just the synchronize() polling loop in isolation.
|
|
Simulates the gap between __recentexperiment and __recentresult.
|
|
"""
|
|
print("\n" + "=" * 72)
|
|
print("ISOLATED SYNCHRONIZE POLLING PROFILE")
|
|
print("=" * 72)
|
|
|
|
data = {"__recentexperiment": 0, "__recentresult": 0}
|
|
quit_flag = threading.Event()
|
|
|
|
# Simulate: experiment advances faster than result
|
|
# Experiment is at job N, result is at job N-before
|
|
# synchronize() must wait for result to catch up
|
|
|
|
total_wait_time = 0
|
|
num_sync_calls = 0
|
|
total_poll_iterations = 0
|
|
|
|
for job_id in range(num_jobs):
|
|
data["__recentexperiment"] = job_id
|
|
|
|
# Simulate result lagging behind by 'before' jobs
|
|
# In real code, result catches up asynchronously
|
|
# We simulate this by having result advance at a fixed rate
|
|
|
|
wait_start = time.perf_counter()
|
|
iterations = 0
|
|
while (data["__recentexperiment"] > data["__recentresult"] + before) and not quit_flag.isSet():
|
|
iterations += 1
|
|
quit_flag.wait(0.1) # the waitsteps parameter
|
|
|
|
# Simulate result catching up (in real code, this happens in ResultHandling)
|
|
if data["__recentresult"] < job_id - before:
|
|
data["__recentresult"] = min(job_id - before, data["__recentresult"] + 1)
|
|
|
|
wait_elapsed = time.perf_counter() - wait_start
|
|
total_wait_time += wait_elapsed
|
|
num_sync_calls += 1
|
|
total_poll_iterations += iterations
|
|
|
|
avg_wait = total_wait_time / num_sync_calls if num_sync_calls else 0
|
|
avg_iterations = total_poll_iterations / num_sync_calls if num_sync_calls else 0
|
|
|
|
print(f"Jobs: {num_jobs}, before: {before}")
|
|
print(f"synchronize() calls: {num_sync_calls}")
|
|
print(f"Total wait time: {total_wait_time*1000:.1f}ms")
|
|
print(f"Avg wait per call: {avg_wait*1000:.1f}ms")
|
|
print(f"Total poll iterations: {total_poll_iterations}")
|
|
print(f"Avg iterations per call: {avg_iterations:.1f}")
|
|
print(f"Poll overhead: ~{total_poll_iterations * 0.1 * 1000:.1f}ms of wake-up latency")
|
|
print()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Profile XML parsing on realistic data
|
|
# ---------------------------------------------------------------------------
|
|
def profile_xml_parsing(num_jobs=100, samples=1024, channels=2, spool_dir=None):
|
|
"""Profile just the XML parsing + base64 decode cost."""
|
|
print("\n" + "=" * 72)
|
|
print("XML PARSING PROFILE")
|
|
print("=" * 72)
|
|
|
|
if spool_dir is None:
|
|
spool_dir = tempfile.mkdtemp(prefix="damaris_xml_profile_")
|
|
|
|
import base64
|
|
import numpy
|
|
|
|
# Generate result files
|
|
for job_id in range(num_jobs):
|
|
result_file = os.path.join(spool_dir, f"job.{job_id:09d}.result")
|
|
with open(result_file, "w") as f:
|
|
f.write(f'<result job="{job_id}">\n')
|
|
f.write(f' <adcdata rate="1000000.0" channels="{channels}" samples="{samples}">\n')
|
|
fake_data = bytes([random.randint(0, 255) for _ in range(samples * channels)])
|
|
encoded = base64.b64encode(fake_data).decode("ascii")
|
|
for i in range(0, len(encoded), 62):
|
|
f.write(encoded[i:i+62] + "\n")
|
|
f.write(f' </adcdata>\n')
|
|
f.write(f'</result>\n')
|
|
|
|
# Profile parsing
|
|
reader = BlockingResultReader(spool_dir)
|
|
parse_times = []
|
|
decode_times = []
|
|
split_times = []
|
|
|
|
for job_id in range(num_jobs):
|
|
result_file = os.path.join(spool_dir, f"job.{job_id:09d}.result")
|
|
|
|
t0 = time.perf_counter()
|
|
result = reader.get_result_object(result_file)
|
|
total = time.perf_counter() - t0
|
|
|
|
# The parsing happens inside get_result_object
|
|
# We can't easily separate XML parse from decode in the current code
|
|
# but we can measure the total
|
|
parse_times.append(total)
|
|
|
|
avg_parse = numpy.mean(parse_times) * 1000
|
|
max_parse = numpy.max(parse_times) * 1000
|
|
total_parse = numpy.sum(parse_times) * 1000
|
|
|
|
print(f"Jobs: {num_jobs}, Samples: {samples}, Channels: {channels}")
|
|
print(f"Total parse time: {total_parse:.1f}ms")
|
|
print(f"Avg per result: {avg_parse:.2f}ms")
|
|
print(f"Max per result: {max_parse:.2f}ms")
|
|
print(f"Throughput: {num_jobs / (total_parse/1000):.0f} results/sec")
|
|
print()
|
|
|
|
shutil.rmtree(spool_dir, ignore_errors=True)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="Profile synchronize() round-trip")
|
|
parser.add_argument("--jobs", type=int, default=100, help="Number of jobs (default: 100)")
|
|
parser.add_argument("--samples", type=int, default=1024, help="Samples per job (default: 1024)")
|
|
parser.add_argument("--channels", type=int, default=2, help="Channels (default: 2)")
|
|
parser.add_argument("--spool", type=str, default=None, help="Spool directory")
|
|
parser.add_argument("--mode", choices=["full", "polling", "xml", "all"], default="all",
|
|
help="Profiling mode (default: all)")
|
|
args = parser.parse_args()
|
|
|
|
if args.mode in ("polling", "all"):
|
|
# Profile synchronize polling with different lag values
|
|
for before in [0, 5, 10, 20]:
|
|
profile_synchronize_polling(num_jobs=args.jobs, before=before)
|
|
|
|
if args.mode in ("xml", "all"):
|
|
profile_xml_parsing(num_jobs=args.jobs, samples=args.samples, channels=args.channels, spool_dir=args.spool)
|
|
|
|
if args.mode in ("full",):
|
|
print("Full threaded profile requires a running backend. Skipping.")
|
|
|
|
if args.mode == "all":
|
|
print("\n" + "=" * 72)
|
|
print("SUMMARY OF FINDINGS")
|
|
print("=" * 72)
|
|
print("""
|
|
The synchronize() round-trip bottleneck analysis:
|
|
|
|
1. POLLING LATENCY (synchronize() + BlockingResultReader)
|
|
- synchronize() polls every 100ms (waitsteps=0.1)
|
|
- BlockingResultReader polls every 100ms (poll_time=0.1)
|
|
- Worst-case added latency: ~200ms per job (one full cycle of each poll)
|
|
- This is the PRIMARY bottleneck for small job counts
|
|
|
|
2. XML PARSING + BASE64 DECODE
|
|
- Per-result overhead depends on sample count
|
|
- For 1024 samples, 2 channels: ~X ms per result
|
|
- Scales linearly with sample count
|
|
|
|
3. FILE I/O
|
|
- Writing job files: minimal (atomic rename)
|
|
- Reading result files: depends on disk speed and result size
|
|
|
|
4. RESULT HANDLING ITERATION
|
|
- Minimal overhead (dict updates + yield)
|
|
|
|
RECOMMENDATION:
|
|
- Reduce synchronize() waitsteps from 0.1 to 0.01 for lower latency
|
|
- Reduce BlockingResultReader poll_time from 0.1 to 0.01
|
|
- Consider using inotify/fsevents for file-system notifications instead of polling
|
|
""")
|