From 013d1162a261001a759fbd873a82cf778a493e45 Mon Sep 17 00:00:00 2001
From: Marko Mecina <marko.mecina@univie.ac.at>
Date: Thu, 27 Oct 2022 14:01:20 +0200
Subject: [PATCH] make S13 data header parameters configurable
---
Ccs/ccs_function_lib.py | 135 +++++++++++++++++++++++++++++-------
Ccs/decompression.py | 9 ++-
Ccs/packet_config_CHEOPS.py | 14 ++++
Ccs/packet_config_SMILE.py | 17 ++++-
4 files changed, 144 insertions(+), 31 deletions(-)
diff --git a/Ccs/ccs_function_lib.py b/Ccs/ccs_function_lib.py
index 2a91934..e6abdeb 100644
--- a/Ccs/ccs_function_lib.py
+++ b/Ccs/ccs_function_lib.py
@@ -57,21 +57,25 @@ except SQLOperationalError as err:
project = cfg.get('ccs-database', 'project')
pc = importlib.import_module(PCPREFIX + str(project).upper())
-PUS_VERSION, TMHeader, TCHeader, PHeader, TM_HEADER_LEN, TC_HEADER_LEN, P_HEADER_LEN, PEC_LEN, MAX_PKT_LEN, timepack, \
- timecal, calc_timestamp, CUC_OFFSET, CUC_EPOCH, crc, PLM_PKT_PREFIX_TC_SEND, PLM_PKT_SUFFIX, FMT_TYPE_PARAM = \
- [pc.PUS_VERSION, pc.TMHeader, pc.TCHeader, pc.PHeader,
- pc.TM_HEADER_LEN, pc.TC_HEADER_LEN, pc.P_HEADER_LEN, pc.PEC_LEN,
- pc.MAX_PKT_LEN, pc.timepack, pc.timecal, pc.calc_timestamp,
- pc.CUC_OFFSET, pc.CUC_EPOCH, pc.puscrc, pc.PLM_PKT_PREFIX_TC_SEND, pc.PLM_PKT_SUFFIX, pc.FMT_TYPE_PARAM]
+# project specific parameters, must be present in all packet_config_* files
+try:
+ PUS_VERSION, TMHeader, TCHeader, PHeader, TM_HEADER_LEN, TC_HEADER_LEN, P_HEADER_LEN, PEC_LEN, MAX_PKT_LEN, timepack, \
+ timecal, calc_timestamp, CUC_OFFSET, CUC_EPOCH, crc, PLM_PKT_PREFIX_TC_SEND, PLM_PKT_SUFFIX, FMT_TYPE_PARAM = \
+ [pc.PUS_VERSION, pc.TMHeader, pc.TCHeader, pc.PHeader,
+ pc.TM_HEADER_LEN, pc.TC_HEADER_LEN, pc.P_HEADER_LEN, pc.PEC_LEN,
+ pc.MAX_PKT_LEN, pc.timepack, pc.timecal, pc.calc_timestamp,
+ pc.CUC_OFFSET, pc.CUC_EPOCH, pc.puscrc, pc.PLM_PKT_PREFIX_TC_SEND, pc.PLM_PKT_SUFFIX, pc.FMT_TYPE_PARAM]
+
+ s13_unpack_data_header = pc.s13_unpack_data_header
+except AttributeError as err:
+ logger.critical(err)
+ raise err
SREC_MAX_BYTES_PER_LINE = 250
SEG_HEADER_LEN = 12
SEG_SPARE_LEN = 2
SEG_CRC_LEN = 2
-SDU_PAR_LENGTH = 1
-S13_HEADER_LEN_TOTAL = 21 # length of PUS + source header in S13 packets (i.e. data to be removed when collecting S13)
-
pid_offset = int(cfg.get('ccs-misc', 'pid_offset'))
fmtlist = {'INT8': 'b', 'UINT8': 'B', 'INT16': 'h', 'UINT16': 'H', 'INT32': 'i', 'UINT32': 'I', 'INT64': 'q',
@@ -803,13 +807,13 @@ def read_stream(stream, fmt, pos=None, offbi=0):
return x
-##
-# csize
-#
-# Returns the Amount of Bytes for the input format
-# @param fmt Input String that defines the format
-# @param offbi
def csize(fmt, offbi=0):
+ """
+ Returns the amount of bytes required for the input format
+ @param fmt: Input String that defines the format
+ @param offbi:
+ @return:
+ """
if fmt in ('i24', 'I24'):
return 3
elif fmt.startswith('uint'):
@@ -821,7 +825,10 @@ def csize(fmt, offbi=0):
elif fmt.startswith('ascii'):
return int(fmt[5:])
else:
- return struct.calcsize(fmt)
+ try:
+ return struct.calcsize(fmt)
+ except struct.error:
+ raise NotImplementedError(fmt)
##
@@ -855,7 +862,7 @@ def none_to_empty(s):
def Tm_header_formatted(tm, detailed=False):
- '''unpack APID, SEQCNT, PKTLEN, TYPE, STYPE, SOURCEID'''
+ """unpack APID, SEQCNT, PKTLEN, TYPE, STYPE, SOURCEID"""
# if len(tm) < TC_HEADER_LEN:
# return 'Cannot decode header - packet has only {} bytes!'.format(len(tm))
@@ -1462,9 +1469,15 @@ def get_cuctime(tml):
return cuc_timestamp
-def get_pool_rows(pool_name, dbcon=None):
+def get_pool_rows(pool_name, check_existence=False):
dbcon = scoped_session_storage
+ if check_existence:
+ check = dbcon.query(DbTelemetryPool).filter(DbTelemetryPool.pool_name == pool_name)
+ if not check.count():
+ dbcon.close()
+ raise ValueError('Pool "{}" does not exist.'.format(pool_name))
+
rows = dbcon.query(
DbTelemetry
).join(
@@ -3037,9 +3050,9 @@ def get_tc_calibration_and_parameters(ccf_descr=None):
return calibrations_dict
-def get_tm_parameter_list(st, sst, apid, pi1val):
- que = 'SELECT pid_spid, pid_tpsd FROM pid WHERE pid_type={} AND pid_stype={} AND pid_apid={} AND pid_pi1_val={}'.format(st, sst, apid, pi1val)
- spid, tpsd = scoped_session_idb.execute(que).fetchall()[0]
+def get_tm_parameter_list(st, sst, apid=None, pi1val=0):
+
+ spid, tpsd = _get_spid(st, sst, apid=apid, pi1val=pi1val)
if tpsd == -1:
que = 'SELECT plf_name, pcf_descr, plf_offby, pcf_ptc, pcf_pfc FROM plf LEFT JOIN pcf ON plf_name=pcf_name WHERE plf_spid={} ORDER BY plf_offby, plf_offbi'.format(spid)
@@ -3052,7 +3065,8 @@ def get_tm_parameter_list(st, sst, apid, pi1val):
def get_tm_parameter_info(pname):
- que = 'SELECT ocp_lvalu, ocp_hvalu, ocp_type, txp_from, txp_altxt FROM pcf LEFT JOIN ocp ON pcf_name=ocp_name LEFT JOIN txp ON pcf_curtx=txp_numbr WHERE pcf_name="{}" ORDER BY txp_from, ocp_pos'.format(pname)
+ que = 'SELECT ocp_lvalu, ocp_hvalu, ocp_type, txp_from, txp_altxt FROM pcf LEFT JOIN ocp ON pcf_name=ocp_name ' \
+ 'LEFT JOIN txp ON pcf_curtx=txp_numbr WHERE pcf_name="{}" ORDER BY txp_from, ocp_pos'.format(pname)
res = scoped_session_idb.execute(que).fetchall()
return res
@@ -3103,6 +3117,55 @@ def get_tm_id(pcf_descr=None):
return tms_dict
+def get_tm_parameter_sizes(st, sst, apid=None, pi1val=0):
+ """
+ Returns a list of parameters and their sizes. For variable length TMs only the first fixed part is considered.
+ @param st:
+ @param sst:
+ @param apid:
+ @param pi1val:
+ @return:
+ """
+
+ spid, tpsd = _get_spid(st, sst, apid=apid, pi1val=pi1val)
+
+ if tpsd == -1:
+ que = 'SELECT plf_name, pcf_descr, pcf_ptc, pcf_pfc, NULL FROM plf LEFT JOIN pcf ON plf_name=pcf_name WHERE plf_spid={} ORDER BY plf_offby, plf_offbi'.format(spid)
+ else:
+ que = 'SELECT vpd_name, pcf_descr, pcf_ptc, pcf_pfc, vpd_grpsize FROM vpd LEFT JOIN pcf ON vpd_name=pcf_name WHERE vpd_tpsd={} ORDER BY vpd_pos'.format(tpsd)
+
+ res = scoped_session_idb.execute(que).fetchall()
+
+ pinfo = []
+ for p in res:
+ pinfo.append((p[1], csize(ptt(*p[2:4]))))
+ # break after first "counter" parameter
+ if p[-1] != 0:
+ break
+
+ return pinfo
+
+
+def _get_spid(st, sst, apid=None, pi1val=0):
+ """
+
+ @param st:
+ @param sst:
+ @param apid:
+ @param pi1val:
+ @return:
+ """
+ if apid is None:
+ apid = ''
+ else:
+ apid = ' AND pid_apid={}'.format(apid)
+
+ que = 'SELECT pid_spid, pid_tpsd FROM pid WHERE pid_type={} AND pid_stype={}{} AND pid_pi1_val={}'.format(st, sst, apid, pi1val)
+ spid, tpsd = scoped_session_idb.execute(que).fetchall()[0]
+
+ return spid, tpsd
+
+
def get_data_pool_items(pcf_descr=None, src_file=None):
if not isinstance(src_file, (str, type(None))):
raise TypeError('src_file must be str, is {}.'.format(type(src_file)))
@@ -4147,13 +4210,15 @@ def collect_13(pool_name, starttime=None, endtime=None, startidx=None, endidx=No
def dump_large_data(pool_name, starttime=0, endtime=None, outdir="", dump_all=False, sdu=None, startidx=None,
endidx=None):
"""
- Dump 13,2 data to disk
- @param endidx:
+ Dump 13,2 data to disk. For pools loaded from a file, pool_name must be the absolute path of that file.
@param pool_name:
@param starttime:
@param endtime:
@param outdir:
@param dump_all:
+ @param sdu:
+ @param startidx:
+ @param endidx:
"""
filedict = {}
ldt_dict = collect_13(pool_name, starttime=starttime, endtime=endtime, join=True, collect_all=dump_all,
@@ -4161,13 +4226,18 @@ def dump_large_data(pool_name, starttime=0, endtime=None, outdir="", dump_all=Fa
for buf in ldt_dict:
if ldt_dict[buf] is None:
continue
- obsid, time, ftime, ctr = struct.unpack('>IIHH', ldt_dict[buf][:12]) # TODO this has to be configurable
+
+ try:
+ obsid, time, ftime, ctr = s13_unpack_data_header(ldt_dict[buf])
+ except ValueError as err:
+ logger.error('Incompatible definition of S13 data header.')
+ raise err
+
fname = os.path.join(outdir, "LDT_{:03d}_{:010d}_{:06d}.ce".format(obsid, time, ctr))
with open(fname, "wb") as fdesc:
fdesc.write(ldt_dict[buf])
filedict[buf] = fdesc.name
- # return list(ldt_dict.keys())
return filedict
@@ -5046,6 +5116,8 @@ class ProjectDialog(Gtk.Dialog):
sys.exit()
+# some default parameter definitions that require functions defined above
+
# create local look-up tables for data pool items from MIB
try:
DP_ITEMS_SRC_FILE = cfg.get('database', 'datapool-items')
@@ -5061,3 +5133,14 @@ except (FileNotFoundError, ValueError, confignator.config.configparser.NoOptionE
finally:
DP_IDS_TO_ITEMS = {int(k[0]): k[1] for k in _dp_items}
DP_ITEMS_TO_IDS = {k[1]: int(k[0]) for k in _dp_items}
+
+# S13 header/offset info
+try:
+ _s13_info = get_tm_parameter_sizes(13, 1)
+ SDU_PAR_LENGTH = _s13_info[0][-1]
+ # length of PUS + source header in S13 packets (i.e. data to be removed when collecting S13)
+ S13_HEADER_LEN_TOTAL = TM_HEADER_LEN + sum([p[-1] for p in _s13_info])
+except (SQLOperationalError, NotImplementedError, IndexError):
+ logger.warning('Could not get S13 info from MIB, using default values')
+ SDU_PAR_LENGTH = 1
+ S13_HEADER_LEN_TOTAL = 21
diff --git a/Ccs/decompression.py b/Ccs/decompression.py
index de58d6d..62af3b8 100644
--- a/Ccs/decompression.py
+++ b/Ccs/decompression.py
@@ -13,10 +13,13 @@ cfg = confignator.get_config(check_interpolation=False)
logger = logging.getLogger(__name__)
logger.setLevel(getattr(logging, cfg.get('ccs-logging', 'level').upper()))
+CE_COLLECT_TIMEOUT = 1
+LDT_MINIMUM_CE_GAP = 0.001
+
ce_decompressors = {}
-def create_fits(data=None, header=None, filename=None):
+def create_fits(header=None, filename=None):
hdulist = pyfits.HDUList()
hdu = pyfits.PrimaryHDU()
hdu.header = header
@@ -117,8 +120,8 @@ class CeDecompress:
self.ce_decompression_on = False
self.ce_thread = None
self.last_ce_time = 0
- self.ce_collect_timeout = 1
- self.ldt_minimum_ce_gap = 0.001
+ self.ce_collect_timeout = CE_COLLECT_TIMEOUT
+ self.ldt_minimum_ce_gap = LDT_MINIMUM_CE_GAP
def _ce_decompress(self):
checkdir = os.path.dirname(self.outdir)
diff --git a/Ccs/packet_config_CHEOPS.py b/Ccs/packet_config_CHEOPS.py
index 3aca67c..5bb405c 100644
--- a/Ccs/packet_config_CHEOPS.py
+++ b/Ccs/packet_config_CHEOPS.py
@@ -8,6 +8,8 @@ Author: Marko Mecina (MM)
import ctypes
import datetime
+import struct
+
from s2k_partypes import ptt
import crcmod
@@ -376,3 +378,15 @@ class EventDetectionData(ctypes.Union):
def __init__(self):
raise NotImplementedError('Not available in project CHEOPS')
+
+
+# S13 data header format, using python struct conventions
+S13_FMT_OBSID = 'I'
+S13_FMT_TIME = 'I'
+S13_FMT_FTIME = 'H'
+S13_FMT_COUNTER = 'H'
+_S13_HEADER_FMT = S13_FMT_OBSID + S13_FMT_TIME + S13_FMT_FTIME + S13_FMT_COUNTER
+
+
+def s13_unpack_data_header(buf):
+ return struct.unpack('>' + _S13_HEADER_FMT, buf[:struct.calcsize(_S13_HEADER_FMT)])
diff --git a/Ccs/packet_config_SMILE.py b/Ccs/packet_config_SMILE.py
index 9034adf..2fba577 100644
--- a/Ccs/packet_config_SMILE.py
+++ b/Ccs/packet_config_SMILE.py
@@ -8,6 +8,7 @@ Author: Marko Mecina (MM)
import ctypes
import datetime
+import struct
import numpy as np
import crcmod
@@ -737,10 +738,10 @@ class FeeDataTransfer(FeeDataTransferHeader):
if self.bits.PKT_TYPE == FEE_PKT_TYPE_EV_DET:
evtdata = EventDetectionData()
evtdata.bin[:] = self.data
+ # structure according to MSSL-SMILE-SXI-IRD-0001
self.evt_data = {"COLUMN": evtdata.bits.column,
"ROW": evtdata.bits.row,
- "IMAGE": np.array(evtdata.bits.array)[
- ::-1]} # structure according to MSSL-SMILE-SXI-IRD-0001
+ "IMAGE": np.array(evtdata.bits.array)[::-1]}
else:
self.evt_data = None
@@ -760,3 +761,15 @@ class EventDetectionData(ctypes.Union):
("bits", EventDetectionFields),
("bin", ctypes.c_ubyte * ctypes.sizeof(EventDetectionFields))
]
+
+
+# S13 data header format, using python struct conventions
+S13_FMT_OBSID = 'I'
+S13_FMT_TIME = 'I'
+S13_FMT_FTIME = 'H'
+S13_FMT_COUNTER = 'H'
+_S13_HEADER_FMT = S13_FMT_OBSID + S13_FMT_TIME + S13_FMT_FTIME + S13_FMT_COUNTER
+
+
+def s13_unpack_data_header(buf):
+ return struct.unpack('>' + _S13_HEADER_FMT, buf[:struct.calcsize(_S13_HEADER_FMT)])
--
GitLab