Public Access
added proper logging facility
This commit is contained in:
markusro committed
1 parent
fa064c1727
commit
0eb44039f0
6 files changed
+315
-53
No files matched your search
@@ -5,12 +5,16 @@ import sys
|
||||
import time
|
||||
import re
|
||||
import glob
|
||||
import logging
|
||||
from . import ExperimentWriter
|
||||
from . import ResultReader
|
||||
from .logging_utils import FileTailerHandler
|
||||
import threading
|
||||
import types
|
||||
import signal
|
||||
|
||||
logger = logging.getLogger("damaris.backend_driver")
|
||||
|
||||
if sys.platform=="win32":
|
||||
import winreg
|
||||
|
||||
@@ -40,7 +44,7 @@ class BackendDriver(threading.Thread):
|
||||
try:
|
||||
os.makedirs(os.path.abspath(self.spool_dir))
|
||||
except OSError as e:
|
||||
print(e)
|
||||
logger.error("Could not create backend's spool directory %s: %s", self.spool_dir, e)
|
||||
raise AssertionError("could not create backend's spool directory %s "%self.spool_dir)
|
||||
|
||||
# remove stale state filenames
|
||||
@@ -62,10 +66,10 @@ class BackendDriver(threading.Thread):
|
||||
if os.path.isdir("/proc/%d"%core_pid):
|
||||
raise AssertionError("found backend with pid %d (state file %s) in same spool dir"%(core_pid,statefilename))
|
||||
else:
|
||||
print("removing stale backend state file", statefilename)
|
||||
logger.warning("Removing stale backend state file: %s", statefilename)
|
||||
os.remove(statefilename)
|
||||
else:
|
||||
print("todo: take care of existing backend state files")
|
||||
logger.warning("TODO: take care of existing backend state files (Windows)")
|
||||
|
||||
self.result_reader = ResultReader.BlockingResultReader(self.spool_dir,
|
||||
no=0,
|
||||
@@ -81,21 +85,9 @@ class BackendDriver(threading.Thread):
|
||||
self.raised_exception=None
|
||||
|
||||
def run(self):
|
||||
# take care of older logfiles
|
||||
self.core_output_filename=os.path.join(self.spool_dir,"logdata")
|
||||
if os.path.isfile(self.core_output_filename):
|
||||
i=0
|
||||
max_logs=100
|
||||
while os.path.isfile(self.core_output_filename+".%02d"%i):
|
||||
i+=1
|
||||
while (i>=max_logs):
|
||||
i-=1
|
||||
os.remove(self.core_output_filename+".%02d"%i)
|
||||
for j in range(i):
|
||||
os.rename(self.core_output_filename+".%02d"%(i-j-1),self.core_output_filename+".%02d"%(i-j))
|
||||
os.rename(self.core_output_filename, self.core_output_filename+".%02d"%0)
|
||||
# create logfile
|
||||
self.core_output=open(self.core_output_filename,"w")
|
||||
# Log file for backend output
|
||||
self.core_output_filename = os.path.join(self.spool_dir, "logdata")
|
||||
self.core_output = open(self.core_output_filename, "w")
|
||||
|
||||
# again look out for existing state files
|
||||
state_files=glob.glob(os.path.join(self.spool_dir,"*.state"))
|
||||
@@ -118,6 +110,10 @@ class BackendDriver(threading.Thread):
|
||||
stdout=self.core_output,
|
||||
stderr=self.core_output)
|
||||
|
||||
# Start tailing backend log file for unified logging
|
||||
self._log_tailer = FileTailerHandler(self.core_output_filename)
|
||||
self._log_tailer.start()
|
||||
|
||||
# wait till state file shows up
|
||||
timeout=10
|
||||
# to do: how should I know core's state name????!!!!!
|
||||
@@ -126,19 +122,16 @@ class BackendDriver(threading.Thread):
|
||||
while len(state_files)==0:
|
||||
if timeout<0 or self.core_input is None or self.core_input.poll() is not None or self.quit_flag.isSet():
|
||||
# look into core log file and include contents
|
||||
print(timeout, self.core_input, self.quit_flag.isSet())
|
||||
if self.core_input is not None:
|
||||
print(self.core_input.poll())
|
||||
logger.error("Backend failed to start: timeout=%s poll=%s", timeout, self.core_input.poll())
|
||||
log_message=''
|
||||
self.core_input=None
|
||||
if os.path.isfile(self.core_output_filename):
|
||||
# to do include log data
|
||||
log_message='\n'+''.join(open(self.core_output_filename,"r", errors='replace').readlines()[:10])
|
||||
if not log_message:
|
||||
log_message=" no error message from core"
|
||||
self.core_output.close()
|
||||
self.raised_exception="no state file appeared or backend died away:"+log_message
|
||||
print(self.raised_exception)
|
||||
logger.error(self.raised_exception)
|
||||
self.quit_flag.set()
|
||||
return
|
||||
time.sleep(0.05)
|
||||
@@ -147,7 +140,7 @@ class BackendDriver(threading.Thread):
|
||||
|
||||
# save the one
|
||||
if len(state_files)>1:
|
||||
print("did find more than one state file, taking first one!")
|
||||
logger.warning("Found more than one state file, taking first one!")
|
||||
self.statefilename=state_files[0]
|
||||
|
||||
# read state file
|
||||
@@ -182,28 +175,28 @@ class BackendDriver(threading.Thread):
|
||||
backend_result=self.core_input.poll()
|
||||
if backend_result is not None: break
|
||||
if wait_loop_counter==10:
|
||||
print("sending termination signal to backend process")
|
||||
logger.warning("Sending termination signal to backend process")
|
||||
self.send_signal("SIGTERM")
|
||||
elif wait_loop_counter==20:
|
||||
print("sending kill signal to backend process")
|
||||
logger.warning("Sending kill signal to backend process")
|
||||
self.send_signal("SIGKILL")
|
||||
elif wait_loop_counter>30:
|
||||
print("no longer waiting for backend shutdown")
|
||||
logger.warning("No longer waiting for backend shutdown")
|
||||
break
|
||||
|
||||
if backend_result is None:
|
||||
print("backend dit not end properly, please stop it manually")
|
||||
logger.error("Backend did not end properly, please stop it manually")
|
||||
elif backend_result>0:
|
||||
print("backend returned ", backend_result)
|
||||
logger.error("Backend returned exit code %d", backend_result)
|
||||
elif backend_result<0:
|
||||
sig_name=[x for x in dir(signal) if x.startswith("SIG") and \
|
||||
x[3]!="_" and \
|
||||
(type(signal.__dict__[x])is int) and \
|
||||
signal.__dict__[x]==-backend_result]
|
||||
if sig_name:
|
||||
print("backend was terminated by signal ",sig_name[0])
|
||||
logger.error("Backend was terminated by signal %s", sig_name[0])
|
||||
else:
|
||||
print("backend was terminated by signal no",-backend_result)
|
||||
logger.error("Backend was terminated by signal no %d", -backend_result)
|
||||
self.core_input = None
|
||||
self.core_pid = None
|
||||
|
||||
@@ -226,12 +219,6 @@ class BackendDriver(threading.Thread):
|
||||
if os.path.isfile(resultfilename):
|
||||
os.remove(resultfilename)
|
||||
|
||||
def get_messages(self):
|
||||
# return pending messages
|
||||
if self.core_output.tell()==os.path.getsize(self.core_output_filename):
|
||||
return None
|
||||
return self.core_output.read()
|
||||
|
||||
def restart_queue(self):
|
||||
self.send_signal("SIGUSR1")
|
||||
|
||||
@@ -250,7 +237,7 @@ class BackendDriver(threading.Thread):
|
||||
|
||||
def send_signal(self, sig):
|
||||
if self.core_pid is None:
|
||||
print("BackendDriver.send_signal is called with core_pid=None")
|
||||
logger.error("BackendDriver.send_signal called with core_pid=None")
|
||||
return
|
||||
try:
|
||||
if sys.platform[:5]=="linux":
|
||||
@@ -262,7 +249,7 @@ class BackendDriver(threading.Thread):
|
||||
kill_command=os.path.join(cygwin_path,"bin","kill.exe")
|
||||
os.popen("%s -%s %d"%(kill_command,sig,self.core_pid))
|
||||
except OSError as e:
|
||||
print("could not send signal %s to core: %s"%(sig, str(e)))
|
||||
logger.error("Could not send signal %s to core: %s", sig, str(e))
|
||||
|
||||
|
||||
def is_busy(self):
|
||||
@@ -294,3 +281,5 @@ class BackendDriver(threading.Thread):
|
||||
if self.core_output:
|
||||
self.core_output.close()
|
||||
self.core_output=None
|
||||
if hasattr(self, '_log_tailer'):
|
||||
self._log_tailer.stop()
|
||||
Reference in new issue
Block a user