diff --git a/Ccs/ccs_function_lib.py b/Ccs/ccs_function_lib.py
index 1677e126af5775932d9e2fb2d6269aadef8d98c0..10bd82c814c8beb6e4eb281bdbfb239107ea40db 100644
--- a/Ccs/ccs_function_lib.py
+++ b/Ccs/ccs_function_lib.py
@@ -330,7 +330,7 @@ def dbus_connection(name, instance=1):
return dbuscon
except:
# print('Please start ' + str(name) + ' if it is not running')
- logger.warning('Connection to ' + str(name) + ' is not possible.')
+ logger.info('Connection to ' + str(name) + ' is not possible.')
return False
@@ -594,8 +594,8 @@ def user_tm_decoders_func():
#
# Return a formatted string containing all the decoded source data of TM packet _tm_
# @param tm TM packet bytestring
-def Tmformatted(tm, separator='\n', sort_by_name=False, textmode=True, UDEF=False):
- sourcedata, tmtcnames = Tmdata(tm, UDEF=UDEF)
+def Tmformatted(tm, separator='\n', sort_by_name=False, textmode=True, udef=False):
+ sourcedata, tmtcnames = Tmdata(tm, udef=udef)
tmtcname = " / ".join(tmtcnames)
if textmode:
if sourcedata is not None:
@@ -624,17 +624,18 @@ def Tmformatted(tm, separator='\n', sort_by_name=False, textmode=True, UDEF=Fals
#
# Decode source data field of TM packet
# @param tm TM packet bytestring
-def Tmdata(tm, UDEF=False, *args):
+def Tmdata(tm, udef=False):
tpsd = None
params = None
dbcon = scoped_session_idb
- # This will be used to first check if an UDEF exists and used this to decode, if not the IDB will be checked
- if UDEF:
+ # check if a UDEF exists and use to decode, if not the IDB will be checked
+ if udef:
try:
header, data, crc = Tmread(tm)
st, sst, apid = header.SERV_TYPE, header.SERV_SUB_TYPE, header.APID
que = 'SELECT pic_pi1_off,pic_pi1_wid from pic where pic_type=%s and pic_stype=%s' % (st, sst)
+ # que = 'SELECT pic_pi1_off,pic_pi1_wid from pic where pic_type=%s and pic_stype=%s and pic_apid=%s' % (st, sst, apid)
dbres = dbcon.execute(que)
pi1, pi1w = dbres.fetchall()[0]
@@ -642,41 +643,46 @@ def Tmdata(tm, UDEF=False, *args):
tag = '{}-{}-{}-{}'.format(st, sst, apid, pi1val)
user_label, params = user_tm_decoders[tag]
spid = None
- #o = data.unpack(','.join([ptt[i[4]][i[5]] for i in params]))
- if len(params[0]) == 9: #Length of a parameter which should be decoded acording to given position
+
+ # Length of a parameter which should be decoded acording to given position
+ if len(params[0]) == 9:
vals_params = decode_pus(data, params)
- else: #Decode according to given order, length is then 11
+ # Decode according to given order, length is then 11
+ else:
vals_params = read_variable_pckt(data, params)
tmdata = [(get_calibrated(i[0], j[0]), i[6], i[1], pidfmt(i[7]), j) for i, j in zip(params, vals_params)]
tmname = ['USER DEFINED: {}'.format(user_label)]
return tmdata, tmname
- except:
- logger.info('UDEF could not be found, search in IDB')
+
+ except Exception as err:
+ logger.info('UDEF could not be found, search in IDB ({})'.format(err))
+ finally:
+ dbcon.close()
try:
if (tm[0] >> 4) & 1:
- return Tcdata(tm, *args)
- # with poolmgr.lock:
+ return Tcdata(tm)
+
header, data, crc = Tmread(tm)
- # data = tm_list[-2]
st, sst, apid = header.SERV_TYPE, header.SERV_SUB_TYPE, header.APID
que = 'SELECT pic_pi1_off,pic_pi1_wid from pic where pic_type=%s and pic_stype=%s' % (st, sst)
dbres = dbcon.execute(que)
pi1, pi1w = dbres.fetchall()[0]
+
if pi1 != -1:
- #print(tm[pi1:pi1 + pi1w])
- # pi1val = Bits(tm)[pi1 * 8:pi1 * 8 + pi1w].uint
pi1val = int.from_bytes(tm[pi1:pi1 + pi1w//8], 'big')
que = 'SELECT pid_spid,pid_tpsd,pid_dfhsize from pid where pid_type=%s and pid_stype=%s and ' \
'pid_apid=%s and pid_pi1_val=%s' % (st, sst, apid, pi1val)
else:
que = 'SELECT pid_spid,pid_tpsd,pid_dfhsize from pid where pid_type=%s and pid_stype=%s and ' \
'pid_apid=%s' % (st, sst, apid)
+
dbres = dbcon.execute(que)
fetch = dbres.fetchall()
+
# if APID or SID does not match:
if len(fetch) != 0:
spid, tpsd, dfhsize = fetch[0]
@@ -696,6 +702,7 @@ def Tmdata(tm, UDEF=False, *args):
if params is None:
dbres = dbcon.execute(que)
spid, tpsd, dfhsize = dbres.fetchall()[0]
+
# TODO: proper handling of super-commutated parameters
if tpsd == -1 and params is None:
que = 'SELECT pcf.pcf_name,pcf.pcf_descr,plf_offby,plf_offbi,pcf.pcf_ptc,pcf.pcf_pfc,\
@@ -708,14 +715,15 @@ def Tmdata(tm, UDEF=False, *args):
tmdata = [(get_calibrated(i[0], j[0]), i[6], i[1], pidfmt(i[7]), j) for i, j in zip(params, vals_params)]
elif params is not None:
- #o = data.unpack(','.join([ptt[i[4]][i[5]] for i in params]))
-
- if len(params[0]) == 9: #Length of a parameter which should be decoded acording to given position
+ # Length of a parameter which should be decoded acording to given position
+ if len(params[0]) == 9:
vals_params = decode_pus(data, params)
- else: #Decode according to given order, length is then 11
+ # Decode according to given order, length is then 11
+ else:
vals_params = read_variable_pckt(data, params)
tmdata = [(get_calibrated(i[0], j[0]), i[6], i[1], pidfmt(i[7]), j) for i, j in zip(params, vals_params)]
+
else:
que = 'SELECT pcf.pcf_name,pcf.pcf_descr,pcf.pcf_ptc,pcf.pcf_pfc,pcf.pcf_curtx,pcf.pcf_width,\
pcf.pcf_unit,pcf.pcf_pid,vpd_pos,vpd_grpsize,vpd_fixrep from vpd left join pcf on \
@@ -733,10 +741,13 @@ def Tmdata(tm, UDEF=False, *args):
tmname = dbres.fetchall()[0]
else:
tmname = ['USER DEFINED: {}'.format(user_label)]
+
except Exception as failure:
raise Exception('Packet data decoding failed: ' + str(failure))
+
finally:
dbcon.close()
+
return tmdata, tmname
@@ -803,7 +814,7 @@ def read_stream(stream, fmt, pos=None, offbi=0):
x = x.decode('ascii')
except UnicodeDecodeError as err:
logger.warning(err)
- x = str(data)
+ x = x.decode('utf-8', errors='replace')
elif fmt == timepack[0]:
x = timecal(data)
else:
@@ -812,7 +823,7 @@ def read_stream(stream, fmt, pos=None, offbi=0):
return x
-def csize(fmt, offbi=0):
+def csize(fmt, offbi=0, bitsize=False):
"""
Returns the amount of bytes required for the input format
@param fmt: Input String that defines the format
@@ -866,6 +877,10 @@ def none_to_empty(s):
return '' if s is None else s
+def str_to_int(itr):
+ return int(itr) if itr.lower() != 'none' else None
+
+
def Tm_header_formatted(tm, detailed=False):
"""unpack APID, SEQCNT, PKTLEN, TYPE, STYPE, SOURCEID"""
@@ -965,13 +980,15 @@ def parameter_tooltip_text(x):
h = hex(x)[3:].upper()
elif isinstance(x, float):
h = struct.pack('>f', x).hex().upper()
+ elif isinstance(x, bytes):
+ return x.hex().upper()
else:
# h = str(x)
return str(x)
return 'HEX: 0x{}\nDEC: {}'.format(h, x)
-def Tcdata(tm, *args):
+def Tcdata(tm):
header, data, crc = Tmread(tm)
st, sst, apid = header.SERV_TYPE, header.SERV_SUB_TYPE, header.APID
dbcon = scoped_session_idb
@@ -1201,7 +1218,7 @@ def read_stream_recursive(tms, parameters, decoded=None):
continue
grp = par[-2]
- if grp is None: # None happens for UDFP, would give error using None
+ if grp is None: # None happens for UDEF
grp = 0
fmt = ptt(par[2], par[3])
@@ -1285,6 +1302,7 @@ def pidfmt_reverse(val):
# @param pcf_name PCF_NAME
# @param rawval Raw value of the parameter
def get_calibrated(pcf_name, rawval, properties=None, numerical=False, dbcon=None):
+
if properties is None:
dbcon = scoped_session_idb
que = 'SELECT pcf.pcf_ptc,pcf.pcf_pfc,pcf.pcf_categ,pcf.pcf_curtx from pcf where pcf_name="%s"' % pcf_name
@@ -1292,7 +1310,7 @@ def get_calibrated(pcf_name, rawval, properties=None, numerical=False, dbcon=Non
fetch = dbres.fetchall()
dbcon.close()
if len(fetch) == 0:
- return rawval if isinstance(rawval, (int, float)) else rawval[0]
+ return rawval if isinstance(rawval, (int, float, str, bytes)) else rawval[0]
ptc, pfc, categ, curtx = fetch[0]
@@ -1419,7 +1437,7 @@ def Tm_filter_st(tmlist, st=None, sst=None, apid=None, sid=None, time_from=None,
# tmlist = [tm for tm in list(tmlist) if ((struct.unpack('>H', tm[:2])[0] & 2047) == apid)]
tmlist = [tm for tm in list(tmlist) if (int.from_bytes(tm[:2], 'big') & 0x7FF) == apid]
- if sid is not None:
+ if sid:
if st is None or sst is None or apid is None:
raise ValueError('Must provide st, sst and apid if filtering by sid')
@@ -1436,6 +1454,65 @@ def Tm_filter_st(tmlist, st=None, sst=None, apid=None, sid=None, time_from=None,
return tmlist
+def filter_rows(rows, st=None, sst=None, apid=None, sid=None, time_from=None, time_to=None, idx_from=None, idx_to=None,
+ tmtc=None, get_last=False):
+ """
+ Filter SQL query object by any of the given arguments, return filtered query.
+ @param rows:
+ @param st:
+ @param sst:
+ @param apid:
+ @param sid:
+ @param time_from:
+ @param time_to:
+ @param idx_from:
+ @param idx_to:
+ @param tmtc:
+ @param get_last:
+ """
+
+ if st is not None:
+ rows = rows.filter(DbTelemetry.stc == st)
+
+ if sst is not None:
+ rows = rows.filter(DbTelemetry.sst == sst)
+
+ if apid is not None:
+ rows = rows.filter(DbTelemetry.apid == apid)
+
+ if sid:
+ if st is None or sst is None or apid is None:
+ raise ValueError('Must provide st, sst and apid if filtering by sid')
+
+ sid_offset, sid_bitlen = get_sid(st, sst, apid)
+ if sid_offset != -1:
+ sid_size = sid_bitlen // 8
+ rows = rows.filter(
+ func.mid(DbTelemetry.data, sid_offset - TM_HEADER_LEN + 1, sid_size) == sid.to_bytes(sid_size, 'big'))
+ else:
+ logger.error('SID ({}) not applicable for {}-{}-{}'.format(sid, st, sst, apid))
+
+ if time_from is not None:
+ rows = rows.filter(func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) >= time_from)
+
+ if time_to is not None:
+ rows = rows.filter(func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) <= time_to)
+
+ if idx_from is not None:
+ rows = rows.filter(DbTelemetry.idx >= idx_from)
+
+ if idx_to is not None:
+ rows = rows.filter(DbTelemetry.idx <= idx_to)
+
+ if tmtc is not None:
+ rows = rows.filter(DbTelemetry.is_tm == tmtc)
+
+ if get_last:
+ rows = rows.order_by(DbTelemetry.idx.desc()).first()
+
+ return rows
+
+
##
# CRC check
#
@@ -1506,9 +1583,8 @@ def get_pool_rows(pool_name, check_existence=False):
return rows
-# get values of parameter from HK packets
-def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False, tmfilter=True, pool_name=None):
-
+# get values of parameter from HK packets
+def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False, tmfilter=True, pool_name=None, mk_array=True):
if param is None:
return
@@ -1524,20 +1600,21 @@ def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False,
dbres = dbcon.execute(que)
name, spid, offby, offbi, ptc, pfc, unit, descr, apid, st, sst, hk, sid = dbres.fetchall()[0]
if not isinstance(tmlist, list):
- tmlist = tmlist.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid,
- func.mid(DbTelemetry.data, get_sid(st, sst, apid)[0] - TM_HEADER_LEN + 1,
- get_sid(st, sst, apid)[1] // 8) == sid.to_bytes(get_sid(st, sst, apid)[1] // 8, 'big')
- ).order_by(DbTelemetry.idx.desc())
- if tmlist is not None:
+ # tmlist = tmlist.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid,
+ # func.mid(DbTelemetry.data, get_sid(st, sst, apid)[0] - TM_HEADER_LEN + 1,
+ # get_sid(st, sst, apid)[1] // 8) == sid.to_bytes(get_sid(st, sst, apid)[1] // 8, 'big')
+ # ).order_by(DbTelemetry.idx.desc())
+ tmlist_rows = filter_rows(tmlist, st=st, sst=sst, apid=apid, sid=sid)
+ if tmlist_rows is not None:
if last > 1:
- tmlist_filt = [tm.raw for tm in tmlist[:last]]
+ tmlist = [tm.raw for tm in tmlist_rows[-last:]]
else:
- tmlist_filt = [tmlist.first().raw]
+ tmlist = [tmlist_rows.order_by(DbTelemetry.idx.desc()).first().raw]
else:
- tmlist_filt = []
+ tmlist = []
else:
sid = None if sid == 0 else sid
- tmlist_filt = Tm_filter_st(tmlist, st=st, sst=sst, apid=apid, sid=sid)[-last:] if tmfilter else tmlist[-last:]
+ # tmlist_filt = Tm_filter_st(tmlist, st=st, sst=sst, apid=apid, sid=sid)[-last:] if tmfilter else tmlist[-last:]
ufmt = ptt(ptc, pfc)
@@ -1550,9 +1627,8 @@ def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False,
hkdescr, st, sst, sid, apid, name, spid, offby, offbi, ptc, pfc, unit, descr, pid = dbres.fetchall()[0]
sid = None if sid == 0 else sid
- tmlist_filt = Tm_filter_st(tmlist, st=st, sst=sst, apid=apid, sid=sid)[-last:] if tmfilter else tmlist[-last:]
-
ufmt = ptt(ptc, pfc)
+ # tmlist_filt = Tm_filter_st(tmlist, st=st, sst=sst, apid=apid, sid=sid)[-last:] if tmfilter else tmlist[-last:]
elif hk.startswith('UDEF|'):
label = hk.replace('UDEF|', '')
@@ -1565,21 +1641,30 @@ def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False,
sst = int(pktkey[1])
apid = int(pktkey[2]) if pktkey[2] != 'None' else None
sid = int(pktkey[3]) if pktkey[3] != 'None' else None
- name, descr, _, offbi, ptc, pfc, unit, _, bitlen = parinfo
-
- offby = sum([x[-1] for x in pktinfo[:pktinfo.index(parinfo)]]) // 8 + TM_HEADER_LEN # +TM_HEADER_LEN for header
- tmlist_filt = Tm_filter_st(tmlist, st, sst, apid, sid)[-last:] if tmfilter else tmlist[-last:]
+ # name, descr, _, offbi, ptc, pfc, unit, _, bitlen = parinfo
+ _, descr, ptc, pfc, curtx, bitlen, _, _, _, _, _ = parinfo
+ unit = None
+ name = None
+ offbi = 0
+
+ offby = sum([x[5] for x in pktinfo[:pktinfo.index(parinfo)]]) // 8 + TM_HEADER_LEN # +TM_HEADER_LEN for header
+ # tmlist_filt = Tm_filter_st(tmlist, st, sst, apid, sid)[-last:] if tmfilter else tmlist[-last:]
ufmt = ptt(ptc, pfc)
else:
userpar = json.loads(cfg[CFG_SECT_PLOT_PARAMETERS][param])
- sid = None if (('SID' not in userpar.keys()) or (userpar['SID'] is None)) else userpar['SID']
- tmlist_filt = Tm_filter_st(tmlist, userpar['ST'], userpar['SST'], apid=userpar['APID'], sid=sid)[-last:] if tmfilter else tmlist[-last:]
+ st = int(userpar['ST'])
+ sst = int(userpar['SST'])
+ apid = int(userpar['APID'])
+ sid = None if (('SID' not in userpar) or (userpar['SID'] is None)) else int(userpar['SID'])
+ # tmlist_filt = Tm_filter_st(tmlist, userpar['ST'], userpar['SST'], apid=userpar['APID'], sid=sid)[-last:] if tmfilter else tmlist[-last:]
offby, ufmt = userpar['bytepos'], userpar['format']
offbi = userpar['offbi'] if 'offbi' in userpar else 0
descr, unit, name = param, None, None
bylen = csize(ufmt)
+ tmlist_filt = Tm_filter_st(tmlist, st=st, sst=sst, apid=apid, sid=sid)[-last:] if tmfilter else tmlist[-last:]
+
if name is not None:
que = 'SELECT pcf.pcf_categ,pcf.pcf_curtx from pcf where pcf_name="%s"' % name
dbres = dbcon.execute(que)
@@ -1600,7 +1685,10 @@ def get_param_values(tmlist=None, hk=None, param=None, last=0, numerical=False,
dbcon.close()
try:
- return np.array(np.array(xy).T, dtype='float'), (descr, unit)
+ if mk_array:
+ return np.array(np.array(xy).T, dtype='float'), (descr, unit)
+ else:
+ return xy, (descr, unit)
except ValueError:
return np.array(xy, dtype='float, U32'), (descr, unit)
@@ -1656,9 +1744,9 @@ def get_module_handle(module_name, instance=1, timeout=5):
if module:
break
else:
- time.sleep(1.)
+ time.sleep(.2)
except dbus.DBusException as err:
- logger.warning(err)
+ logger.info(err)
module = False
time.sleep(0.5)
@@ -2138,9 +2226,10 @@ def get_sid(st, sst, apid):
return SID_LUT[(st, sst, apid)]
else:
try:
+ logger.warning('APID {} not known'.format(apid))
return SID_LUT[(st, sst, None)]
except KeyError:
- return None
+ return
##
@@ -2505,28 +2594,36 @@ def Tcsend_bytes(tc_bytes, pool_name='LIVE', pmgr_handle=None):
# Send command to C&C socket
# @param pool_name Name of the pool bound to the socket for CnC/TC communication
# @param cmd Command string to be sent to C&C socket
-def CnCsend(cmd, pool_name=None):
+def CnCsend(cmd, pool_name=None, apid=1804):
global counters # One can only Change variable as global since we are static
-
- pmgr = dbus_connection('poolmanager', communication['poolmanager'])
+ # pmgr = dbus_connection('poolmanager', communication['poolmanager'])
+ pmgr = get_module_handle('poolmanager')
if pool_name is None:
pool_name = pmgr.Variables('tc_name')
- packed_data = CnCpack(data=cmd, sc=counters.setdefault(1804, 1))
+ pid = (apid >> 4) & 0x7F
+ cat = apid & 0xF
+ packed_data = CnCpack(data=cmd, pid=pid, cat=cat, sc=counters.setdefault(apid, 1))
- logger.info('[CNC sent:]' + str(packed_data))
received = pmgr.Functions('socket_send_packed_data', packed_data, pool_name, signature='says')
- if received is not None:
- counters[1804] += 1
+ logger.info('[CNC sent:]' + str(packed_data))
+
+ try:
+ msg = bytes(received)
+ except TypeError as err:
+ logger.error(err)
+ return
+
+ if msg:
+ counters[apid] += 1
try:
- received = bytes(received, encoding='utf-8', errors='replace')
+ msg = msg.decode('ascii', errors='replace')
except Exception as err:
logger.error(err)
- return None
-
- logger.info('[CNC response:]' + received.decode(encoding='utf-8', errors='replace'))
+ return
- return received
+ logger.info('[CNC response:] ' + msg)
+ return msg
##
@@ -2542,6 +2639,10 @@ def CnCsend(cmd, pool_name=None):
# @param gflags Segmentation flags
# @param sc Sequence counter
def CnCpack(data=b'', version=0b011, typ=1, dhead=0, pid=112, cat=12, gflags=0b11, sc=0):
+
+ if isinstance(data, str):
+ data = data.encode('ascii')
+
header = PHeader()
header.bits.PKT_VERS_NUM = version
header.bits.PKT_TYPE = typ
@@ -2551,7 +2652,7 @@ def CnCpack(data=b'', version=0b011, typ=1, dhead=0, pid=112, cat=12, gflags=0b1
header.bits.PKT_SEQ_CNT = sc
header.bits.PKT_LEN = len(data) - 1
- return bytes(header.bin) + data.encode()
+ return bytes(header.bin) + data
##
@@ -3465,6 +3566,12 @@ def get_data_pool_items(pcf_descr=None, src_file=None, as_dict=False):
return data_pool_dict
+def get_dp_fmt_info(dp_name):
+ que = 'SELECT pcf_name FROM pcf where pcf_pid is not NULL and pcf_descr="{}"'.format(dp_name)
+ mib_name = scoped_session_idb.execute(que).fetchall()[0]
+ return mib_name
+
+
# def get_dp_items(source='mib'):
# fmt = {3: {4: 'UINT8', 12: 'UINT16', 14: 'UINT32'}, 4: {4: 'INT8', 12: 'INT16', 14: 'INT32'}, 5: {1: 'FLOAT'}, 9: {18: 'CUC'}, 7: {1: '1OCT'}}
#
@@ -4278,7 +4385,7 @@ def extract_spw(stream):
header.bits.PKT_TYPE == 0 and header.bits.WRITE == 1):
pktsize = hsize
else:
- pktsize = hsize + header.bits.DATA_LEN# + pc.RMAP_PEC_LEN # TODO: no data CRC from FEEsim?
+ pktsize = hsize + header.bits.DATA_LEN + pc.RMAP_PEC_LEN # TODO: no data CRC from FEEsim?
while len(buf) < pktsize:
data = stream.read(pktsize - len(buf))
diff --git a/Ccs/monitor.py b/Ccs/monitor.py
index 0a43954e81a28777f4bb0b8a06ce8ff690c19570..7636522c2d686493c9256123ea71bde5dd33655f 100644
--- a/Ccs/monitor.py
+++ b/Ccs/monitor.py
@@ -54,8 +54,7 @@ class ParameterMonitor(Gtk.Window):
self.events = {'Error LOW': [(5, 2), 0], 'Error MEDIUM': [(5, 3), 0], 'Error HIGH': [(5, 4), 0]}
self.evt_reset_values = {'Error LOW': 0, 'Error MEDIUM': 0, 'Error HIGH': 0}
- # self.box = Gtk.VBox()
- # self.menubar = Gtk.MenuBar()
+ self.evt_pkt_idx_last = 0 # last packet idx up to which the evts were counted
self.grid = Gtk.Grid()
self.grid.set_column_homogeneous(True)
@@ -64,10 +63,12 @@ class ParameterMonitor(Gtk.Window):
hbox = Gtk.HBox()
self.add(hbox)
+ self._res_evt_cnt_callback = None
self.evt_cnt = self.create_event_counter()
self.evt_check_enabled = True
self.evt_check_tocnt = 0
+ self.pool_id = None # used to track "clear pool" events
self.pool_name = pool_name
self.parameter_set = parameter_set
self.parameters = {}
@@ -78,8 +79,8 @@ class ParameterMonitor(Gtk.Window):
hbox.pack_start(self.grid, 1, 1, 0)
# Add Univie Button
- univie_box = self.create_univie_box()
- hbox.pack_start(univie_box, 0, 1, 0)
+ # univie_box = self.create_univie_box()
+ # hbox.pack_start(univie_box, 0, 1, 0)
hbox.set_spacing(20)
@@ -95,7 +96,8 @@ class ParameterMonitor(Gtk.Window):
def check_for_pools(self):
try:
- poolmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ # poolmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ poolmgr = cfl.get_module_handle('poolmanager')
pools = poolmgr.Functions('loaded_pools_export_func')
if len(pools) == 1:
pool_name = pools[0][0]
@@ -151,9 +153,11 @@ class ParameterMonitor(Gtk.Window):
evt_cnt.pack_start(box, 0, 0, 0)
reset_button = Gtk.Button(label='Reset')
- reset_button.connect('clicked', self.reset_evt_cnt)
+ reset_button.set_tooltip_text('Reset warning threshold to current # of evt pkts')
+ self._res_evt_cnt_callback = reset_button.connect('clicked', self.reset_evt_cnt)
- set_button = Gtk.Button(label='Set Parameter')
+ set_button = Gtk.Button(label='Set Parameters')
+ set_button.set_tooltip_text('Select/create set of monitored parameters')
set_button.connect('clicked', self.add_evt_cnt)
evt_cnt.pack_start(reset_button, 0, 0, 0)
@@ -233,9 +237,12 @@ class ParameterMonitor(Gtk.Window):
self.pdescr = {x[0]: x[1] for x in descrs}
if self.cfg.has_option('ccs-monitor_parameter_sets', parameter_set):
parameters = json.loads(self.cfg['ccs-monitor_parameter_sets'][parameter_set])
- self.setup_grid(parameters)
+ try:
+ self.setup_grid(parameters)
+ except KeyError as err:
+ self.logger.error('Failed to load parameter set "{}" ({})'.format(parameter_set, err))
else:
- self.logger.warning('Parameter set "{}" does not exist'.format(parameter_set))
+ self.logger.error('Parameter set "{}" does not exist'.format(parameter_set))
def setup_grid(self, parameters):
@@ -246,39 +253,75 @@ class ParameterMonitor(Gtk.Window):
for ncol, col in enumerate(parameters):
for nrow, parameter in enumerate(col):
box = Gtk.HBox()
- parameter, *pktid = eval(parameter)
- box.pktid = tuple(pktid)
- dbres = dbcon.execute('SELECT pcf.pcf_name,pcf.pcf_descr,pcf.pcf_categ,pcf.pcf_unit,ocf.ocf_nbool,\
- ocp.ocp_lvalu,ocp.ocp_hvalu from pcf left join ocf on pcf.pcf_name=ocf.ocf_name\
- left join ocp on ocf_name=ocp_name where pcf.pcf_name="{}"'.format(parameter))
+ parinfo = eval(parameter)
+ if len(parinfo) == 2:
+ parameter, pktid = parinfo[0], eval(parinfo[1])
+ else:
+ parameter, *pktid = parinfo
+
+ if parameter.startswith('UDEF:'):
+ box.pktid = tuple(pktid)
+ parameter, udtype = parameter.split(':')[1:]
+ if udtype == 'dp_item':
+ dbres = dbcon.execute('SELECT pcf.pcf_name,pcf.pcf_descr,pcf.pcf_categ,pcf.pcf_unit,ocf.ocf_nbool,\
+ ocp.ocp_lvalu,ocp.ocp_hvalu from pcf left join ocf on pcf.pcf_name=ocf.ocf_name\
+ left join ocp on ocf_name=ocp_name where pcf.pcf_descr="{}" and pcf_pid is not NULL'.format(parameter))
+ boxdata = dbres.fetchall()
+ if not boxdata:
+ udid = '{}:{}'.format(udtype, cfl.DP_ITEMS_TO_IDS[parameter])
+ boxdata = [[udid, parameter, 'N', None, None, None, None]]
+ elif udtype in ['user_defined', 'user_defined_nopos']:
+ udid = '{}:{}'.format(udtype, parameter)
+ boxdata = [[udid, parameter, 'N', None, None, None, None]]
+ else:
+ dbres = dbcon.execute('SELECT pcf.pcf_name,pcf.pcf_descr,pcf.pcf_categ,pcf.pcf_unit,ocf.ocf_nbool,\
+ ocp.ocp_lvalu,ocp.ocp_hvalu from pcf left join ocf on pcf.pcf_name=ocf.ocf_name\
+ left join ocp on ocf_name=ocp_name where pcf.pcf_name="{}"'.format(udtype))
+ boxdata = dbres.fetchall()
+
+ elif parameter.startswith('user_defined'):
+ udtype, parameter = parameter.split(':')
+ box.pktid = ((pktid['bytepos'], pktid['offbi'], pktid['format']), pktid['ST'], pktid['SST'],
+ pktid['APID'], pktid['SID'], None, None)
+ udid = '{}:{}'.format(udtype, parameter)
+ boxdata = [[udid, parameter, 'N', None, None, None, None]]
+
+ else:
+ box.pktid = tuple(pktid)
+ dbres = dbcon.execute('SELECT pcf.pcf_name,pcf.pcf_descr,pcf.pcf_categ,pcf.pcf_unit,ocf.ocf_nbool,\
+ ocp.ocp_lvalu,ocp.ocp_hvalu from pcf left join ocf on pcf.pcf_name=ocf.ocf_name\
+ left join ocp on ocf_name=ocp_name where pcf.pcf_name="{}"'.format(parameter))
+ boxdata = dbres.fetchall()
- boxdata = dbres.fetchall()
try:
nlims = boxdata[0][-3]
if nlims in (1, None):
- box.param_id, plabel, box.format, unit, _, lolim, hilim = boxdata[0]
+ box.param_id, plabel, categ, unit, _, lolim, hilim = boxdata[0]
hardlim = (lolim, hilim)
softlim = (None, None)
else:
- box.param_id, plabel, box.format, unit = boxdata[0][:4]
+ box.param_id, plabel, categ, unit = boxdata[0][:4]
softlim, hardlim = [(x[-2], x[-1]) for x in boxdata]
+ box.format = self.get_fstr(box.param_id, plabel, categ=categ)
except IndexError:
self.logger.error('Parameter {} does not exist - cannot add!'.format(parameter))
continue
+
# override with user defined limits
- if self.pdescr.get(parameter) in self.user_limits:
+ if self.pdescr.get(parameter, parameter) in self.user_limits:
try:
- softlim = self.user_limits[self.pdescr[parameter]]['soft']
+ softlim = self.user_limits[self.pdescr.get(parameter, parameter)]['soft']
except KeyError:
softlim = (None, None)
- hardlim = self.user_limits[self.pdescr[parameter]]['hard']
+ hardlim = self.user_limits[self.pdescr.get(parameter, parameter)]['hard']
box.limits = (softlim, hardlim)
pname, pvalue = Gtk.Label(), Gtk.TextView()
- pname.set_markup('<span size="large" weight="bold">{} [{}]</span>'.format(plabel, unit))
+ unit_txt = '[{}]'.format(unit) if unit is not None else ''
+ pname.set_markup('<span size="large" weight="bold">{} {}</span>'.format(plabel, unit_txt))
pname.set_xalign(0)
- pname.set_tooltip_text(box.param_id)
+ pname.set_tooltip_text(box.param_id.split(':')[-1])
buf = Gtk.TextBuffer()
buf.insert_markup(buf.get_start_iter(),
@@ -312,8 +355,10 @@ class ParameterMonitor(Gtk.Window):
self.monitored_pkts = {self.parameters[k]['pktid']: {'pkttime': 0, 'reftime': time.time(), 'data': None} for k in self.parameters}
+ # LUT for user defined parameter names by pktid
+ self.pname_from_pktid = {self.parameters[k]['pktid']: k.split(':')[-1] for k in self.parameters}
+
self.grid.show_all()
- return
def update_parameter_view(self, interval=INTERVAL, max_age=MAX_AGE):
self.interval = interval
@@ -332,25 +377,23 @@ class ParameterMonitor(Gtk.Window):
start = time.time()
self.update_parameters_worker()
dt = time.time() - start
- # print(dt)
+ # print('#', threading.enumerate(), dt)
+ if dt > self.interval:
+ self.logger.warning('Monitoring cycle takes longer than requested interval ({:.3f} > {:.3f})!'.format(dt, self.interval))
+ # self.disable_evt_cnt()
time.sleep(self.interval - min(self.interval, dt))
def update_parameters_worker(self):
rows = cfl.get_pool_rows(self.pool_name)
- if self.evt_check_enabled:
- ctime = time.time()
- self.check_evts(rows)
- cdt = time.time() - ctime
- # disable check_evts if it causes too much delay
- if cdt > 0.7 * self.interval:
- self.evt_check_tocnt += 1
- if self.evt_check_tocnt > 5:
- self.disable_evt_cnt()
+ try:
+ pool_id = rows.first().pool_id # keep track of monitored pool
+ except AttributeError:
+ self.logger.debug('No rows in pool yet')
+ return
for pktid in self.monitored_pkts:
pktinfo = self.get_last_pkt_with_id(rows, pktid)
-
if pktinfo is None:
continue
@@ -359,8 +402,18 @@ class ParameterMonitor(Gtk.Window):
self.monitored_pkts[pktid]['reftime'] = time.time()
self.monitored_pkts[pktid]['pkttime'] = pkttime
- tm = cfl.Tmdata(pkt)[0]
- self.monitored_pkts[pktid]['data'] = {par[4][1][0]: par[0] for par in tm}
+ if isinstance(pktid[0], tuple):
+ pname = self.pname_from_pktid[pktid]
+ xy, par = cfl.get_param_values(tmlist=[pkt], hk='User defined', param=pname, last=1, tmfilter=False, mk_array=False)
+ udtype = 'user_defined'
+ self.monitored_pkts[pktid]['data'] = {'{}:{}'.format(udtype, par[0]): xy[0][1]}
+ else:
+ try:
+ tm = cfl.Tmdata(pkt)[0]
+ self.monitored_pkts[pktid]['data'] = {self.get_param_id(par): par[0] for par in tm}
+ except Exception as err:
+ self.logger.info('{} {}'.format(err, pktid))
+ continue
checktime = time.time()
for pname in self.parameters:
@@ -399,13 +452,36 @@ class ParameterMonitor(Gtk.Window):
'<span size="large" foreground="{}" weight="bold">{}</span>'.format(
self.parameters[par]['alarm'], '--'), -1)
else:
- buf.insert_markup(buf.get_start_iter(),
- '<span size="large" foreground="{}" weight="bold">{:{fstr}}</span>'.format(
- self.parameters[par]['alarm'], self.parameters[par]['value'],
- fstr=self.parameter_types[self.parameters[par]['format']]), -1)
+ val = self.parameters[par]['value']
+ if isinstance(val, bytes):
+ txt = '<span size="large" foreground="{}" weight="bold">0x{}</span>'.format(
+ self.parameters[par]['alarm'], val.hex().upper())
+ else:
+ txt = '<span size="large" foreground="{}" weight="bold">{:{fstr}}</span>'.format(
+ self.parameters[par]['alarm'], val, fstr=self.parameters[par]['format'])
+
+ buf.insert_markup(buf.get_start_iter(), txt, -1)
GLib.idle_add(updt_buf)
+ if self.evt_check_enabled:
+ ctime = time.time()
+ self.check_evts(rows, incremental=True)
+ cdt = time.time() - ctime
+ # disable check_evts if it causes too much delay
+ if cdt > (0.7 * self.interval):
+ self.evt_check_tocnt += 1
+ if self.evt_check_tocnt > 5:
+ self.disable_evt_cnt()
+
+ if (pool_id != self.pool_id) and (self.pool_id is not None):
+ self.logger.warning('Monitored pool has changed ({})'.format(pool_id))
+ self.evt_pkt_idx_last = 0
+ self.events = {'Error LOW': [(5, 2), 0], 'Error MEDIUM': [(5, 3), 0], 'Error HIGH': [(5, 4), 0]}
+ self.evt_reset_values = {'Error LOW': 0, 'Error MEDIUM': 0, 'Error HIGH': 0}
+
+ self.pool_id = pool_id
+
# def updt_bg_color():
# alarms = [self.parameters[x]['alarm'] for x in self.parameters.keys()]
# if alarms.count('red'):
@@ -425,25 +501,59 @@ class ParameterMonitor(Gtk.Window):
# GLib.idle_add(updt_bg_color)
# return
- def get_last_pkt_with_id(self, rows, pktid):
+ def get_last_pkt_with_id(self, rows, pktid, pidx=0):
spid, st, sst, apid, pi1, pi1off, pi1wid = pktid
- if pi1off != -1:
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid,
- func.mid(DbTelemetry.data, pi1off - cfl.TM_HEADER_LEN + 1, pi1wid // 8) == pi1.to_bytes(
- pi1wid // 8, 'big')).order_by(DbTelemetry.idx.desc()).first()
+ if pi1off != -1: # and (pi1off is not None):
+ # rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid,
+ # func.mid(DbTelemetry.data, pi1off - cfl.TM_HEADER_LEN + 1, pi1wid // 8) == pi1.to_bytes(
+ # pi1wid // 8, 'big'), DbTelemetry.idx>=pidx).order_by(DbTelemetry.idx.desc()).first()
+ rows = cfl.filter_rows(rows, st=st, sst=sst, apid=apid, sid=pi1, idx_from=pidx).order_by(DbTelemetry.idx.desc()).first()
else:
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid).order_by(DbTelemetry.idx.desc()).first()
-
+ # rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid, DbTelemetry.idx>=pidx).order_by(DbTelemetry.idx.desc()).first()
+ rows = cfl.filter_rows(rows, st=st, sst=sst, apid=apid, idx_from=pidx).order_by(DbTelemetry.idx.desc()).first()
if rows is None:
return
return float(rows.timestamp[:-1]), rows.raw
- def pckt_counter(self, rows, st, sst):
- npckts = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst).count()
+ def get_param_id(self, pinfo):
+ """
+ Get param_id from parinfo as returned by Tmdata for look-up in monitored packets data
+ @param par:
+ """
+ par = pinfo[4][1]
+
+ if par[0] == 'dp_item':
+ return '{}:{}'.format(par[0], cfl.DP_ITEMS_TO_IDS[par[1]])
+ elif par[0] in ['user_defined', 'user_defined_nopos']:
+ return '{}:{}'.format(par[0], par[1])
+ else:
+ return par[0]
+
+ def get_fstr(self, param_id, name, categ=None):
+
+ if categ == 'S':
+ return 's'
+
+ udtype = param_id.split(':')[0]
+ pinfo = cfl._parameter_decoding_info((name, udtype), check_curtx=True)
+
+ ptc, pfc = pinfo[2:4]
+
+ if ptc in [7]:
+ return ''
+ elif ptc in [5, 9]:
+ return '.13G'
+ elif ptc in [8]:
+ return 's'
+ else:
+ return 'd'
+
+ def pckt_counter(self, rows, st, sst, pidx=0):
+ npckts = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.idx > pidx).count()
return npckts
- def check_evts(self, rows):
+ def check_evts(self, rows, incremental=True):
def updt_buf(buf, evt):
buf.delete(*buf.get_bounds())
buf.insert_markup(buf.get_start_iter(),
@@ -454,18 +564,23 @@ class ParameterMonitor(Gtk.Window):
for event in self.evt_cnt.get_children()[:-2]:
evt = event.get_children()[0].get_text()
- self.events[evt][1] = self.pckt_counter(rows, *self.events[evt][0])
+ if incremental:
+ self.events[evt][1] += self.pckt_counter(rows, *self.events[evt][0], pidx=self.evt_pkt_idx_last)
+ else:
+ self.events[evt][1] = self.pckt_counter(rows, *self.events[evt][0])
buf = event.get_children()[1].get_buffer()
GLib.idle_add(updt_buf, buf, evt)
- def updt_bg_color():
- if self.events['Error HIGH'][1] > self.evt_reset_values['Error HIGH']:
- self.override_background_color(Gtk.StateType.NORMAL, self.alarm_colors['red'])
- self.present()
- elif self.events['Error MEDIUM'][1] > self.evt_reset_values['Error MEDIUM']:
- self.override_background_color(Gtk.StateType.NORMAL, self.alarm_colors['orange'])
+ self.evt_pkt_idx_last = rows.order_by(DbTelemetry.idx.desc()).first().idx
+
+ # def updt_bg_color():
+ # if self.events['Error HIGH'][1] > self.evt_reset_values['Error HIGH']:
+ # self.override_background_color(Gtk.StateType.NORMAL, self.alarm_colors['red'])
+ # self.present()
+ # elif self.events['Error MEDIUM'][1] > self.evt_reset_values['Error MEDIUM']:
+ # self.override_background_color(Gtk.StateType.NORMAL, self.alarm_colors['orange'])
# GLib.idle_add(updt_bg_color)
@@ -494,9 +609,33 @@ class ParameterMonitor(Gtk.Window):
rbutton = self.evt_cnt.get_children()[-2]
rbutton.set_label('Count EVTs')
- rbutton.set_tooltip_text('Event counting has been disabled because of heavy load, probably because of a too large pool.\nClick to force count update.')
+ rbutton.set_tooltip_text('Event counting has been disabled because of heavy load, probably because of a too large pool. Click to force count update.')
+ rbutton.disconnect(self._res_evt_cnt_callback)
+ self._res_evt_cnt_callback = rbutton.connect('clicked', self.reenable_evt_cnt)
self.logger.warning('Counting events takes too long - disabling.')
+ def reenable_evt_cnt(self, widget):
+ self.evt_check_enabled = True
+ self.evt_check_tocnt = 0
+
+ def updt_buf(cbuf, cevt):
+ cbuf.delete(*cbuf.get_bounds())
+ cbuf.insert_markup(cbuf.get_start_iter(), '<span size="large" foreground="{}" weight="bold">{}</span>'.format(
+ 'black', self.events[cevt][1]), -1)
+
+ for evt in self.evt_cnt.get_children()[:-2]:
+ evt.set_sensitive(True)
+ buf = evt.get_children()[1]
+
+ GLib.idle_add(updt_buf, buf.get_buffer(), evt.get_children()[0].get_text())
+
+ widget.set_label('Reset')
+ widget.set_tooltip_text('Reset warning threshold to current # of evt pkts')
+ widget.disconnect(self._res_evt_cnt_callback)
+ self._res_evt_cnt_callback = widget.connect('clicked', self.reset_evt_cnt)
+
+ self.logger.info('Event counting re-enabled')
+
def add_evt_cnt(self, widget=None):
self.monitor_setup()
@@ -510,9 +649,9 @@ class ParameterMonitor(Gtk.Window):
def monitor_setup(self, parameter_set=None, nslots=3):
if parameter_set is not None:
- parameters = json.loads(self.cfg['ccs-monitor_parameter_sets'][parameter_set])
- self.setup_grid(parameters)
-
+ # parameters = json.loads(self.cfg['ccs-monitor_parameter_sets'][parameter_set])
+ # self.setup_grid(parameters)
+ self.set_parameter_view(parameter_set)
return
dialog = MonitorSetupDialog(logger=self.logger, nslots=nslots, parameter_set=parameter_set, parent=self)
@@ -838,6 +977,7 @@ class MonitorSetupDialog(Gtk.Dialog):
hks = dbres.fetchall()
topleveliters = {}
+ # packets in MIB
for hk in hks:
if not hk[2] in topleveliters:
@@ -854,13 +994,29 @@ class MonitorSetupDialog(Gtk.Dialog):
params = dbres.fetchall()
- [parameter_model.append(it, [par[0], str(par[1:])]) for par in params]
+ for par in params:
+ parameter_model.append(it, [par[0], str(par[1:])])
dbcon.close()
+ # UDEF packets
+ udpkts = self.monitor.cfg['ccs-user_defined_packets']
+ self.useriter = parameter_model.append(None, ['UDEF packets', None])
+ for userpacket in udpkts:
+ st, sst, apid, sid = map(cfl.str_to_int, userpacket.split('-'))
+ sid_off, sid_bitlen = cfl.get_sid(st, sst, apid)
+ pktdef = json.loads(udpkts[userpacket])
+ pktiter = parameter_model.append(self.useriter, [pktdef[0], None])
+ for userpar in pktdef[1]:
+ name = 'UDEF:{}:{}'.format(userpar[1], userpar[0])
+ parameter_model.append(pktiter, [userpar[1], str([name, None, st, sst, apid, sid, sid_off, sid_bitlen])])
+
+ # user-defined stand-alone ("plot") parameters
self.useriter = parameter_model.append(None, ['User defined', None])
for userpar in self.monitor.cfg['ccs-plot_parameters']:
- parameter_model.append(self.useriter, [userpar, None])
+ name = 'user_defined:{}'.format(userpar)
+ parameter_model.append(self.useriter, [userpar, str([name,
+ self.monitor.cfg['ccs-plot_parameters'][userpar]])])
return parameter_model
@@ -904,22 +1060,13 @@ class MonitorSetupDialog(Gtk.Dialog):
for slots in param_set:
i += 1
pnames = {eval(par)[0]: par for par in slots}
- if len(pnames) > 1:
- dbres = dbcon.execute('SELECT pcf.pcf_descr, pcf.pcf_name FROM pcf where pcf.pcf_name in {}'.format(tuple(pnames)))
- params = dbres.fetchall()
- elif len(pnames) == 1:
- dbres = dbcon.execute('SELECT pcf.pcf_descr, pcf.pcf_name FROM pcf where pcf.pcf_name="{}"'.format(pnames[0]))
- params = dbres.fetchall()
- else:
- continue
-
- for par in params:
- self.slots[i][3].append([par[0], pnames[par[1]]])
+ for par in pnames:
+ self.slots[i][3].append([par.split(':')[1], pnames[par]])
dbcon.close()
else:
- self.logger.info('Given Set name could not be found in config File')
+ self.logger.error('Parameter set {} not found'.format(entry))
if __name__ == "__main__":
@@ -953,6 +1100,10 @@ if __name__ == "__main__":
elif len(sys.argv) == 1:
is_pool = win.check_for_pools()
+ else:
+ win.quit_func()
+ sys.exit()
+
if is_pool == 0:
win.quit_func()
sys.exit()
diff --git a/Ccs/plotter.py b/Ccs/plotter.py
index a4c833b481409f70ddc69a400c88a1b1ce957694..2413dc654bab42576786bb4584c94525c6f4f9d5 100644
--- a/Ccs/plotter.py
+++ b/Ccs/plotter.py
@@ -69,6 +69,7 @@ class PlotViewer(Gtk.Window):
self.parameter_limits = set()
self.data_dict = {}
+ self.data_dict_info = {} # row idx of last data point in data_dict
self.max_datapoints = 0
self.data_min_idx = None
self.data_max_idx = None
@@ -253,7 +254,7 @@ class PlotViewer(Gtk.Window):
def _create_navbar(self):
# navbar = NavigationToolbarX(self.canvas, self)
- navbar = NavigationToolbar(self.canvas, self)
+ navbar = NavigationToolbar(self.canvas, window=self)
limits = Gtk.HBox()
self.xmin = Gtk.Entry()
@@ -438,8 +439,10 @@ class PlotViewer(Gtk.Window):
# Or check between which Pools should be selected
all_pools = None
if cfl.is_open('poolmanager'):
- pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
- all_pools = cfl.Dictionaries(pmgr, 'loaded_pools')
+ # pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ pmgr = cfl.get_module_handle('poolmanager')
+ all_pools = pmgr.Dictionaries('loaded_pools')
+ # all_pools = cfl.Dictionaries(pmgr, 'loaded_pools')
#if not all_pools:
# found_pools = None
#elif len(active_pool) == 1:
@@ -451,7 +454,9 @@ class PlotViewer(Gtk.Window):
elif cfl.is_open('poolviewer'):
pv = cfl.dbus_connection('poolviewer', cfl.communication['poolviewer'])
- all_pools = cfl.Variables(pv, 'active_pool_info')
+ pv = cfl.get_module_handle('poolviewer')
+ all_pools = pv.Variables('active_pool_info')
+ # all_pools = cfl.Variables(pv, 'active_pool_info')
#if all_pools:
#loaded_pool = ActivePoolInfo(active_pool[0],active_pool[1],active_pool[2],active_pool[3])
# found_pools = all_pools[2]
@@ -669,6 +674,10 @@ class PlotViewer(Gtk.Window):
else:
selection = self.treeview.get_selection()
model, treepath = selection.get_selected()
+
+ if treepath is None:
+ return
+
parameter = model[treepath][0]
if model[treepath].parent is None:
@@ -677,15 +686,14 @@ class PlotViewer(Gtk.Window):
hk = model[treepath].parent[0]
rows = cfl.get_pool_rows(self.loaded_pool.filename)
-
- # TODO filter by idx
rows = self.set_plot_range(rows)
dbcon = self.session_factory_idb
if hk != 'User defined' and not hk.startswith('UDEF|'):
- que = 'SELECT pid_type,pid_stype,pid_pi1_val,pid_apid FROM pid LEFT JOIN plf ON pid.pid_spid=plf.plf_spid LEFT JOIN pcf ' \
- 'ON plf.plf_name=pcf.pcf_name WHERE pcf.pcf_descr="{}" AND pid.pid_descr="{}"'.format(parameter, hk)
+ que = 'SELECT pid_type,pid_stype,pid_pi1_val,pid_apid FROM pid LEFT JOIN plf ON pid.pid_spid=plf.plf_spid ' \
+ 'LEFT JOIN pcf ON plf.plf_name=pcf.pcf_name WHERE pcf.pcf_descr="{}" AND ' \
+ 'pid.pid_descr="{}"'.format(parameter, hk)
dbres = dbcon.execute(que).fetchall()
if not dbres:
@@ -694,18 +702,8 @@ class PlotViewer(Gtk.Window):
st, sst, sid, apid = dbres[0]
- sid_pos = self.sid_position_query(st, sst, apid, sid)
-
- if sid_pos is not None:
- sid_offset, sid_size = sid_pos
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid,
- func.mid(DbTelemetry.data, sid_offset - TM_HEADER_LEN + 1, sid_size) ==
- sid.to_bytes(sid_size, 'big'))
- else:
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid)
-
elif hk.startswith('UDEF|'):
- label = hk.strip('UDEF|')
+ label = hk.replace('UDEF|', '')
tag = [k for k in self.user_tm_decoders if self.user_tm_decoders[k][0] == label][0]
pktinfo = tag.split('-')
st = int(pktinfo[0])
@@ -713,32 +711,27 @@ class PlotViewer(Gtk.Window):
apid = int(pktinfo[2]) if pktinfo[2] != 'None' else None
sid = int(pktinfo[3]) if pktinfo[3] != 'None' else None
- sid_pos = self.sid_position_query(st, sst, apid, sid)
-
- if sid_pos is not None:
- sid_offset, sid_size = sid_pos
-
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.sst == apid,
- func.mid(DbTelemetry.data, sid_offset - TM_HEADER_LEN + 1, sid_size) ==
- sid.to_bytes(sid_size, 'big'))
- else:
- rows = rows.filter(DbTelemetry.stc == st, DbTelemetry.sst == sst, DbTelemetry.apid == apid)
else:
userpar = json.loads(self.cfg[cfl.CFG_SECT_PLOT_PARAMETERS][parameter])
- rows = rows.filter(DbTelemetry.stc == userpar['ST'], DbTelemetry.sst == userpar['SST'],
- DbTelemetry.apid == userpar['APID'])
+ st, sst, apid = userpar['ST'], userpar['SST'], userpar['APID']
+
if 'SID' in userpar and userpar['SID']:
- try:
- sid_offset, sid_size = self.sid_position_query(userpar['ST'], userpar['SST'], userpar['APID'], userpar['SID'])
- except TypeError:
- self.logger.error('{}: SID not applicable.'.format(parameter))
- return
+ sid = userpar['SID']
+ else:
+ sid = None
+
+ if self.sid_position_query(st, sst, apid, sid) is None:
+ if sid:
+ self.logger.error('{}: SID not applicable.'.format(parameter))
+ return
+
+ sid = None
- rows = rows.filter(func.mid(DbTelemetry.data, sid_offset - TM_HEADER_LEN + 1, sid_size) ==
- userpar['SID'].to_bytes(sid_size, 'big'))
+ rows = cfl.filter_rows(rows, st=st, sst=sst, apid=apid, sid=sid)
if not self.filter_tl2.get_active():
- rows = rows.filter(func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) > 2.)
+ rows = cfl.filter_rows(rows, time_from=2.)
+ # rows = rows.filter(func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) > 2.)
try:
xy, (descr, unit) = cfl.get_param_values([row.raw for row in rows.yield_per(1000)], hk, parameter,
@@ -746,15 +739,25 @@ class PlotViewer(Gtk.Window):
if len(xy) == 0:
return
- except (ValueError, TypeError):
+ except (ValueError, TypeError) as err:
+ self.logger.debug(err)
self.logger.error("Can't plot {}".format(parameter))
return
+ # store packet info for update worker
self.data_dict[hk + ':' + descr] = xy
+ self.data_dict_info[hk + ':' + descr] = {}
+ self.data_dict_info[hk + ':' + descr]['idx_last'] = rows.order_by(DbTelemetry.idx.desc()).first().idx
+ self.data_dict_info[hk + ':' + descr]['st'] = st
+ self.data_dict_info[hk + ':' + descr]['sst'] = sst
+ self.data_dict_info[hk + ':' + descr]['apid'] = apid
+ self.data_dict_info[hk + ':' + descr]['sid'] = sid
+
# npoints = self.count_datapoints(self.subplot.get_xlim(), self.subplot.get_ylim())
# if npoints > self.max_datapoints > 0:
# xy = xy.T[::npoints // self.max_datapoints + 1].T
self.subplot.autoscale(enable=not self.scaley.get_active(), axis='y')
+
try:
if self.plot_diff.get_active():
x, y = xy
@@ -856,19 +859,21 @@ class PlotViewer(Gtk.Window):
# self.max_datapoints = n
def reduce_datapoints(self, xlim, ylim, fulldata=True):
+
ax = self.canvas.figure.get_axes()[0]
- n_datapoints = self.count_datapoints(xlim, ylim)
- if n_datapoints > self.max_datapoints > 0:
- red_fac = n_datapoints // self.max_datapoints + 1
- for line in ax.lines:
- if not line.get_label().startswith('_lim_'):
- x, y = self.data_dict[line.get_gid() + ':' + line.get_label()]
- if self.plot_diff.get_active():
- x = x[1:]
- y = np.diff(y)
- line.set_xdata(x[::red_fac])
- line.set_ydata(y[::red_fac])
+ if self.max_datapoints > 0:
+ n_datapoints = self.count_datapoints(xlim, ylim)
+ if n_datapoints > self.max_datapoints:
+ red_fac = n_datapoints // self.max_datapoints + 1
+ for line in ax.lines:
+ if not line.get_label().startswith('_lim_'):
+ x, y = self.data_dict[line.get_gid() + ':' + line.get_label()]
+ if self.plot_diff.get_active():
+ x = x[1:]
+ y = np.diff(y)
+ line.set_xdata(x[::red_fac])
+ line.set_ydata(y[::red_fac])
elif fulldata:
for line in ax.lines:
if not line.get_label().startswith('_lim_'):
@@ -890,6 +895,7 @@ class PlotViewer(Gtk.Window):
def clear_parameter(self, widget):
self.data_dict.clear()
+ self.data_dict_info.clear()
self.parameter_limits.clear()
self.subplot.clear()
self.subplot.grid()
@@ -902,7 +908,8 @@ class PlotViewer(Gtk.Window):
def update_plot_worker(self, plot=None, parameter=None):
# pool_name = self.pool_box.get_active_text()
- rows = cfl.get_pool_rows(self.loaded_pool.filename).filter(DbTelemetry.stc == 3, DbTelemetry.sst == 25)
+ rows = cfl.get_pool_rows(self.loaded_pool.filename)
+ rows = self.set_plot_range(rows)
# xmin, xmax = self.subplot.get_xlim()
lines = self.subplot.lines
@@ -912,28 +919,22 @@ class PlotViewer(Gtk.Window):
hk = line.get_gid()
xold, yold = self.data_dict[hk + ':' + parameter]
- time_last = round(float(xold[-1]), 6) # np.float64 not properly understood in sql comparison below
- new_rows = rows.filter(
- func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) > time_last)
+ # time_last = round(float(xold[-1]), 6) # np.float64 not properly understood in sql comparison below
+ # new_rows = rows.filter(func.left(DbTelemetry.timestamp, func.length(DbTelemetry.timestamp) - 1) > time_last)
+ pinfo = self.data_dict_info[hk + ':' + parameter]
+ new_rows = cfl.filter_rows(rows, st=pinfo['st'], sst=pinfo['sst'], apid=pinfo['apid'],
+ sid=pinfo['sid'], idx_from=pinfo['idx_last'] + 1)
+
try:
- xnew, ynew = cfl.get_param_values([row.raw for row in new_rows], hk, parameter,
- numerical=True)[0]
+ # xnew, ynew = cfl.get_param_values([row.raw for row in new_rows], hk, parameter, numerical=True)[0]
+ xnew, ynew = cfl.get_param_values([row.raw for row in new_rows], hk, parameter, numerical=True, tmfilter=False)[0]
+ idx_new = new_rows.order_by(DbTelemetry.idx.desc()).first().idx
except ValueError:
continue
xy = np.stack([np.append(xold, xnew), np.append(yold, ynew)], -1).T
self.data_dict[hk + ':' + parameter] = xy
-
- # line.set_data(xy)
-
- # npoints = xy.shape[1]
- # if npoints > self.max_datapoints > 0:
- # line.set_data(xy.T[::npoints//self.max_datapoints].T)
- # else:
- # line.set_data(xy)
- # line.set_xdata(np.append(xold, xnew))
- # line.set_ydata(np.append(yold, ynew))
- # self.subplot.draw_artist(line)
+ self.data_dict_info[hk + ':' + parameter]['idx_last'] = idx_new
self.reduce_datapoints(self.subplot.get_xlim(), self.subplot.get_ylim())
@@ -1119,7 +1120,8 @@ class PlotViewer(Gtk.Window):
#if self.loaded_pool:
# return
if cfl.is_open('poolviewer'):
- pv = cfl.dbus_connection('poolviewer', cfl.communication['poolviewer'])
+ # pv = cfl.dbus_connection('poolviewer', cfl.communication['poolviewer'])
+ pv = cfl.get_module_handle('poolviewer')
active_pool = cfl.Variables(pv, 'active_pool_info')
#active_pool = pv.Variables('active_pool_info')
if active_pool and active_pool[0]:
@@ -1131,7 +1133,8 @@ class PlotViewer(Gtk.Window):
# self.loaded_pool = None
elif cfl.is_open('poolmanager'):
- pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ # pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ pmgr = cfl.get_module_handle('poolmanager')
active_pool = cfl.Dictionaries(pmgr, 'loaded_pools')
#active_pool = pmgr.Dictionaries('loaded_pools')
#if not active_pool:
@@ -1162,7 +1165,6 @@ class PlotViewer(Gtk.Window):
#if self.loaded_pool:
# self.select_pool(pool=self.loaded_pool)
-
def quit_func(self, *args):
# Try to tell terminal in the editor that the variable is not longer availabe
for service in dbus.SessionBus().list_names():
@@ -1407,7 +1409,8 @@ class SelectPoolDialog(Gtk.Dialog):
# Or check between which Pools should be selected
if cfl.is_open('poolmanager'):
- pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ # pmgr = cfl.dbus_connection('poolmanager', cfl.communication['poolmanager'])
+ pmgr = cfl.get_module_handle('poolmanager')
self.all_pools = pmgr.Dictionaries('loaded_pools')
if not self.all_pools:
self.loaded_pool = None
@@ -1419,7 +1422,8 @@ class SelectPoolDialog(Gtk.Dialog):
self.loaded_pool = list(self.all_pools.keys())
elif cfl.is_open('poolviewer'):
- pv = cfl.dbus_connection('poolviewer', cfl.communication['poolmanager'])
+ # pv = cfl.dbus_connection('poolviewer', cfl.communication['poolmanager'])
+ pv = cfl.get_module_handle('poolviewer')
self.all_pools = pv.Variables('active_pool_info')
if self.all_pools:
#loaded_pool = ActivePoolInfo(active_pool[0],active_pool[1],active_pool[2],active_pool[3])
diff --git a/Ccs/poolview_sql.py b/Ccs/poolview_sql.py
index 79a77a877d7f7a7376a556d12db9e575346acdf3..76dd8479074c54560eb1327689dc000a279edeb3 100644
--- a/Ccs/poolview_sql.py
+++ b/Ccs/poolview_sql.py
@@ -2430,8 +2430,6 @@ class TMPoolView(Gtk.Window):
self.set_tm_data_view()
- return
-
def create_decoder_model(self):
model = Gtk.ListStore(str)
@@ -2461,7 +2459,8 @@ class TMPoolView(Gtk.Window):
def add_decoder(self, widget, decoder_name, byteoffset, bytelength):
try:
label, bytepos, bytelen = decoder_name.get_active_text(), int(byteoffset.get_text()), int(bytelength.get_text())
- except:
+ except Exception as err:
+ self.logger.info(err)
return
if label in (None, ''):
@@ -2608,7 +2607,7 @@ class TMPoolView(Gtk.Window):
datamodel.clear()
try:
if self.UDEF:
- data = cfl.Tmformatted(tm, textmode=False, UDEF=True)
+ data = cfl.Tmformatted(tm, textmode=False, udef=True)
buf = Gtk.TextBuffer(text=cfl.Tm_header_formatted(tm) + '\n{}\n'.format(data[1]))
self._feed_tm_data_view_model(datamodel, data[0])
else:
@@ -3096,10 +3095,6 @@ class TMPoolView(Gtk.Window):
return [row[1] for row in res]
def plot_parameters(self, widget=None, parameters=None, start_live=False):
- #if self.active_pool_info is None:
- # self.logger.warning('Cannot open plot window without pool!')
- # print('Cannot open plot window without pool!')
- # return
cfl.start_plotter(pool_name=self.active_pool_info.pool_name)
diff --git a/Ccs/pus_datapool.py b/Ccs/pus_datapool.py
index 1dd350c373104baee6f8e1c2de97e3d3c638805a..3574e8a6e6553175e76af14db34a7b15452ae1a2 100644
--- a/Ccs/pus_datapool.py
+++ b/Ccs/pus_datapool.py
@@ -822,7 +822,7 @@ class DatapoolManager:
else:
self.decode_tmdump_and_process_packets_internal(buf, process_tm, checkcrc=False)
except socket.timeout as e:
- self.logger.info('Socket timeout ({}:{})'.format(host, port))
+ self.logger.debug('Socket timeout ({}:{})'.format(host, port))
new_session.commit()
continue
except socket.error as e:
@@ -1573,20 +1573,20 @@ class DatapoolManager:
def socket_send_packed_data(self, packdata, poolname):
cncsocket = self.tc_connections[poolname]['socket']
cncsocket.send(packdata)
- received = None
+ received = b''
try:
received = cncsocket.recv(MAX_PKT_LEN)
# self.logger.info.write(logtf(self.tnow()) + ' ' + recv[6:].decode() + ' [CnC]\n')
- self.logger.info(received.decode(errors='replace') + ' [CnC]')
+ self.logger.info(received.decode('utf-8', errors='replace') + ' [CnC]')
# logfile.flush()
# s.close()
# self.counters[1804] += 1
except socket.timeout:
- self.logger.error('Got a timeout')
- self.logger.exception(socket.timeout)
+ # self.logger.error('Got a timeout')
+ self.logger.error(socket.timeout)
# Dbus does not like original data type
- if received is not None:
+ if received:
received = dbus.ByteArray(received)
return received
@@ -1761,7 +1761,7 @@ class DatapoolManager:
header.bits.PKT_TYPE == 0 and header.bits.WRITE == 1):
pktsize = hsize
else:
- pktsize = hsize + header.bits.DATA_LEN# + RMAP_PEC_LEN # TODO: data CRC from FEEsim?
+ pktsize = hsize + header.bits.DATA_LEN + RMAP_PEC_LEN # TODO: data CRC from FEEsim?
while len(buf) < pktsize:
d = sockfd.recv(pktsize - len(buf))