From d12f93447eee5c90b397a6b7404d1a5d38f31bd9 Mon Sep 17 00:00:00 2001
From: Marko Mecina <marko.mecina@univie.ac.at>
Date: Tue, 30 Aug 2022 16:53:47 +0200
Subject: [PATCH] fix bugs in datapool saving
---
Ccs/ccs_function_lib.py | 31 +++++++++---------
Ccs/poolview_sql.py | 70 +++++++++++++++++++----------------------
2 files changed, 50 insertions(+), 51 deletions(-)
diff --git a/Ccs/ccs_function_lib.py b/Ccs/ccs_function_lib.py
index 681f094..fe216c9 100644
--- a/Ccs/ccs_function_lib.py
+++ b/Ccs/ccs_function_lib.py
@@ -1412,12 +1412,12 @@ def get_txp_altxt(pcf_name, alval, dbcon=None):
# @param tmlist List of TM packets
# @param mode Save as "binary" file or "hex" values with one packet per line
# @param st_filter Save only packets of this service type
-def Tmdump(filename, tmlist, mode='hex', st_filter=None, crccheck=False):
+def Tmdump(filename, tmlist, mode='hex', st_filter=None, check_crc=False):
if st_filter is not None:
tmlist = Tm_filter_st(tmlist, *st_filter)
- if crccheck:
- tmlist = (tm for tm in tmlist if not crccheck(tm))
+ if check_crc:
+ tmlist = (tm for tm in tmlist if not crc_check(tm))
if mode.lower() == 'hex':
with open(filename, 'w') as f:
@@ -1440,6 +1440,7 @@ def Tmdump(filename, tmlist, mode='hex', st_filter=None, crccheck=False):
with open(filename, 'w') as f:
f.write('\n'.join(txtlist))
+
##
# Filter by service (sub-)type
#
@@ -1455,35 +1456,37 @@ def Tm_filter_st(tmlist, st=None, sst=None, apid=None, sid=None, time_from=None,
if (st is not None) and (sst is not None):
tmlist = [tm for tm in tmlist if ((tm[7], tm[8]) == (st, sst))]
- if sid != None:
+ if sid is not None:
tmlist = [tm for tm in list(tmlist) if (tm[TM_HEADER_LEN] == sid or tm[TM_HEADER_LEN] + tm[TM_HEADER_LEN + 1] == sid)] # two possibilities for SID because of different definition (length) for SMILE and CHEOPS
- if apid != None:
+ if apid is not None:
tmlist = [tm for tm in list(tmlist) if ((struct.unpack('>H', tm[:2])[0] & 2047) == (apid))]
- if eventId != None:
+ if eventId is not None:
tmlist = [tm for tm in list(tmlist) if (struct.unpack('>H', tm[TM_HEADER_LEN:TM_HEADER_LEN + 2])[0] == eventId)]
- if procId != None:
+ if procId is not None:
tmlist = [tm for tm in list(tmlist) if
(struct.unpack('>H', tm[TM_HEADER_LEN + 2:TM_HEADER_LEN + 4])[0] == procId)]
- if time_from != None:
+ if time_from is not None:
tmlist = [tm for tm in list(tmlist) if (time_from <= get_cuctime(tm))]
- if time_to != None:
+ if time_to is not None:
tmlist = [tm for tm in list(tmlist) if (get_cuctime(tm) <= time_to)]
return tmlist
+
##
# CRC check
#
# Perform a CRC check on the _packet_. Returns True if CRC!=0.
# @param packet TM/TC packet or any bytestring or bitstring object to be CRCed.
def crc_check(packet):
- #if isinstance(packet, (BitArray, BitStream, Bits, ConstBitStream)):
- # packet = packet.bytes
+ """
+ This function returns True if the CRC result is non-zero
+ """
return bool(crc(packet))
@@ -1521,6 +1524,7 @@ def get_cuctime(tml):
return cuc_timestamp
+
def get_pool_rows(pool_name, dbcon=None):
dbcon = scoped_session_storage
@@ -3669,12 +3673,11 @@ def savepool(filename, pool_name, mode='binary', st_filter=None):
logger.info('Saving pool content to disk...')
tmlist = list(get_packets_from_pool(pool_name))
- Tmdump(filename, tmlist, mode=mode, st_filter=st_filter, crccheck=False)
+ Tmdump(filename, tmlist, mode=mode, st_filter=st_filter, check_crc=False)
logger.info('Pool {} saved as {} in {} mode'.format(pool_name, filename, mode.upper()))
- return
-def get_packets_from_pool(pool_name, indices=[], st=None, sst=None, apid=None, dbsession=None):
+def get_packets_from_pool(pool_name, indices=[], st=None, sst=None, apid=None, **kwargs):
"""
@param pool_name:
@param indices:
diff --git a/Ccs/poolview_sql.py b/Ccs/poolview_sql.py
index 4a07c38..4e1835c 100644
--- a/Ccs/poolview_sql.py
+++ b/Ccs/poolview_sql.py
@@ -2612,7 +2612,7 @@ class TMPoolView(Gtk.Window):
self.tm_data_view.thaw_child_notify()
def save_pool(self, widget):
- dialog = SavePoolDialog(parent=self)
+ dialog = SavePoolDialog(parent=self, decoding_type=self.decoding_type)
response = dialog.run()
@@ -2642,7 +2642,7 @@ class TMPoolView(Gtk.Window):
tmlist = self.get_packets_from_indices(filtered=dialog.save_filtered.get_active(), merged_tables=merge)
crc = dialog.crccheck.get_active()
- cfl.Tmdump(filename, tmlist, mode=mode, st_filter=None, crccheck=crc)
+ cfl.Tmdump(filename, tmlist, mode=mode, st_filter=None, check_crc=crc)
self.logger.info(
'{} packets from {} saved as {} in {} mode (CRC {})'.format(len(list(tmlist)),
self.get_active_pool_name(),
@@ -2653,32 +2653,24 @@ class TMPoolView(Gtk.Window):
dialog.destroy()
def save_pool_in_db(self, filename, timestamp, indices=None):
- new_session = self.session_factory_storage
- #new_session.execute(
- # 'delete tm from tm inner join tm_pool on tm.pool_id=tm_pool.iid where tm_pool.pool_name="{}"'.format(
- # filename))
-
+ # this function only works for PUS packets at the moment
+ if self.decoding_type != 'PUS':
+ self.logger.error('Save to DB not supported for {} protocol.'.format(self.decoding_type))
+ return
- new_session.query(
- Telemetry[self.decoding_type]
- ).join(
- DbTelemetryPool,
- Telemetry[self.decoding_type].pool_id == DbTelemetryPool.iid
- ).filter(
- DbTelemetryPool.pool_name == self.active_pool_info.filename
- ).delete()
+ new_session = self.session_factory_storage
+ self.logger.info('Deleting any existing DB entries associated with {}'.format(filename))
+ new_session.execute(
+ 'DELETE tm FROM tm INNER JOIN tm_pool ON tm.pool_id=tm_pool.iid WHERE tm_pool.pool_name="{}"'.format(
+ filename))
+ new_session.execute('DELETE tm_pool FROM tm_pool WHERE tm_pool.pool_name="{}"'.format(filename))
new_session.commit()
+ self.logger.debug('...deleted')
+ self.logger.info('Storing current pool {} in DB for {}'.format(self.active_pool_info.pool_name, filename))
+
+ newPoolRow = DbTelemetryPool(pool_name=filename, protocol='PUS', modification_time=timestamp)
- new_session.query(
- DbTelemetryPool
- ).filter(
- DbTelemetryPool.pool_name == filename
- ).delete()
- new_session.commit()
- newPoolRow = DbTelemetryPool(
- pool_name=filename,
- modification_time=timestamp)
new_session.add(newPoolRow)
new_session.flush()
rows = new_session.query(
@@ -2692,20 +2684,22 @@ class TMPoolView(Gtk.Window):
rows = rows.filter(Telemetry[self.decoding_type].idx.in_(indices))
for idx, row in enumerate(rows, 1):
new_session.add(Telemetry[self.decoding_type](pool_id=newPoolRow.iid,
- idx=idx,
- is_tm=row.is_tm,
- apid=row.apid,
- seq=row.seq,
- len_7=row.len_7,
- stc=row.stc,
- sst=row.sst,
- destID=row.destID,
- timestamp=row.timestamp,
- data=row.data,
- raw=row.raw))
+ idx=idx,
+ is_tm=row.is_tm,
+ apid=row.apid,
+ seq=row.seq,
+ len_7=row.len_7,
+ stc=row.stc,
+ sst=row.sst,
+ destID=row.destID,
+ timestamp=row.timestamp,
+ data=row.data,
+ raw=row.raw))
+
# new_session.flush()
new_session.commit()
new_session.close()
+
'''
# Poolmgr can call the LoadInfo Window via dbus, needed for the load_pool function
def LoadInfo(self):
@@ -2813,7 +2807,7 @@ class TMPoolView(Gtk.Window):
# package_type defines which type was selected by the user, if any was selected
new_pool = cfl.Functions(poolmgr, 'load_pool_poolviewer', pool_name, filename, isbrute, force_db_import,
- self.count_current_pool_rows(), self.my_bus_name[-1], package_type)
+ self.count_current_pool_rows(), self.my_bus_name[-1], package_type)
dialog.destroy()
@@ -3326,7 +3320,7 @@ class ExtractionDialog(Gtk.MessageDialog):
class SavePoolDialog(Gtk.FileChooserDialog):
- def __init__(self, parent=None):
+ def __init__(self, parent=None, decoding_type='PUS'):
super(SavePoolDialog, self).__init__(title="Save packets", parent=parent, action=Gtk.FileChooserAction.SAVE)
self.add_buttons(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
Gtk.STOCK_SAVE, Gtk.ResponseType.OK)
@@ -3353,6 +3347,8 @@ class SavePoolDialog(Gtk.FileChooserDialog):
self.crccheck.set_tooltip_text('Save only packets that pass CRC')
self.store_in_db = Gtk.CheckButton.new_with_label('Store in DB')
self.store_in_db.set_tooltip_text('Permanently store pool in DB - THIS MAY TAKE A WHILE FOR LARGE DATASETS!')
+ if decoding_type != 'PUS':
+ self.store_in_db.set_sensitive(False)
self.save_filtered = Gtk.CheckButton.new_with_label('Apply packet filter')
self.save_filtered.set_tooltip_text('Save only packets according to the currently active poolview filter')
self.merge_tables = Gtk.CheckButton.new_with_label('Merge tables')
--
GitLab