diff --git a/CHANGELOG.md b/CHANGELOG.md
index 82a8b865939b518e5ace05a6ada9e5d731d5d54c..18386d43e4eb8acb3468acf9d855a11e4a916584 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,8 +13,10 @@ separator is added after every second hexadecimal numeric character. Comments
after a '#' symbol until the end of the line are ignored.
E.g. "# comment\n ABCD 1 2\n34B 12\n" are interpreted as {0xAB, 0xCD,
0x01, 0x02, 0x34, 0x0B, 0x12}.
-### Fixed
-- Fix a bug in the definition in imagette header
+### Changed
+- update the header definition according to PLATO-UVIE-PL-UM-0001 Draft 6
+ - changed version_id from 16 to 32 bit in the generic header. Add spare bits to the adaptive imagette header and the non-imagette header, so that the compressed data start address is 4 byte-aligned.
+- Rename cmp_tool_lib.c to cmp_io.c
## [0.07] - 13-12-2021
- **NOTE:** The behaviour of the cmp_tool has changed. From now on, the compressed data will be preceded by a header by default. The old behaviour can be achieved with the `--no_header` option.
diff --git a/README.md b/README.md
index 2b2802ff04774a5e48eebeb156a19ee2fa127026..325a24c78f8767358abc5604d6463f301955ecf9 100644
--- a/README.md
+++ b/README.md
@@ -75,9 +75,11 @@ RDCU-compatible compression mode depending on if the Model (-m) option is set.
This command creates the file `myguess.cfg` with the guessed compression parameters.
### Data Format
+The input data is formatted as hexadecimal numbers.
+For example: `12 AB 34 CD` or `12AB34CD`.
-The input data must be formatted as two hex numbers separated by a space.
-For example: `12 AB 34 CD`.
+### User Manual
+You can find the user manual [here](doc).
## How to use the tool
diff --git a/cmp_tool.c b/cmp_tool.c
index 0cf2acd961580d5fabcd8d0798fab7ef7e2f1bff..fd8401f96975fd06a4c82b7a0216758ab33aae2a 100755
--- a/cmp_tool.c
+++ b/cmp_tool.c
@@ -482,21 +482,29 @@ fail:
static enum cmp_ent_data_type cmp_ent_map_cmp_mode_data_type(uint32_t cmp_mode)
{
+ enum cmp_ent_data_type data_type;
+
switch (cmp_mode) {
case MODE_RAW:
- return DATA_TYPE_IMAGETTE;
case MODE_MODEL_ZERO:
case MODE_DIFF_ZERO:
case MODE_MODEL_MULTI:
case MODE_DIFF_MULTI:
if (print_rdcu_cfg)
- return DATA_TYPE_IMAGETTE_ADAPTIVE;
+ data_type = DATA_TYPE_IMAGETTE_ADAPTIVE;
else
- return DATA_TYPE_IMAGETTE;
+ data_type = DATA_TYPE_IMAGETTE;
+ break;
default:
printf("No mapping between compression mode and header data type\n!");
return DATA_TYPE_UNKOWN;
}
+
+ /* set raw bit if needed */
+ if (raw_mode_is_used(cmp_mode))
+ data_type |= 1UL << RAW_BIT_DATA_TYPE_POS;
+
+ return data_type;
}
diff --git a/doc/PLATO-UVIE-PL-UM-0001_0r6.pdf b/doc/PLATO-UVIE-PL-UM-0001_0r6.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..581a47061c76f787b75e04ccf88a5064f370704e
Binary files /dev/null and b/doc/PLATO-UVIE-PL-UM-0001_0r6.pdf differ
diff --git a/include/cmp_entity.h b/include/cmp_entity.h
index 45a1b61ca837fda4ddf3d3fb60ce26fb53b3bd42..2d6bf542f1ec5b0c25f63fd257a2c3fbb325f78d 100644
--- a/include/cmp_entity.h
+++ b/include/cmp_entity.h
@@ -58,13 +58,13 @@ enum cmp_ent_data_type {
DATA_TYPE_F_CAM_IMAGETTE_ADAPTIVE,
DATA_TYPE_F_CAM_OFFSET,
DATA_TYPE_F_CAM_BACKGROUND,
- DATA_TYPE_UNKOWN = -1
+ DATA_TYPE_UNKOWN = 0x7FFF,
};
-#define GENERIC_HEADER_SIZE 30
+#define GENERIC_HEADER_SIZE 32
#define SPECIFIC_IMAGETTE_HEADER_SIZE 4
-#define SPECIFIC_IMAGETTE_ADAPTIVE_HEADER_SIZE 10
-#define SPECIFIC_NON_IMAGETTE_HEADER_SIZE 30 /* TBC */
+#define SPECIFIC_IMAGETTE_ADAPTIVE_HEADER_SIZE 12
+#define SPECIFIC_NON_IMAGETTE_HEADER_SIZE 32
#define IMAGETTE_HEADER_SIZE \
(GENERIC_HEADER_SIZE + SPECIFIC_IMAGETTE_HEADER_SIZE)
@@ -75,9 +75,9 @@ enum cmp_ent_data_type {
#define CMP_ENTITY_MAX_SIZE 0xFFFFFFUL
-#define RAW_BIT_IN_DATA_TYPE 15U
+#define RAW_BIT_DATA_TYPE_POS 15U
-#define CMP_TOOL_VERSION_ID_BIT 0x8000U /* TBC */
+#define CMP_TOOL_VERSION_ID_BIT 0x80000000U
__extension__
struct timestamp_cmp_ent {
@@ -100,6 +100,7 @@ struct imagette_header {
uint16_t ap2_spill_used; /* Adaptive Spillover threshold used 2 */
uint8_t ap2_golomb_par_used; /* Adaptive Golomb parameter used 2 */
uint8_t spare2;
+ uint16_t spare3;
uint8_t ap_ima_cmp_data[]; /* compressed data for adaptive imagette specific header */
} __attribute__((packed));
};
@@ -120,13 +121,14 @@ struct non_imagette_header {
uint16_t cmp_par_5_used; /* compression parameter 5 used */
uint32_t spill_6_used:24; /* spillover threshold 6 used */
uint16_t cmp_par_6_used; /* compression parameter 6 used */
+ uint16_t spare;
uint8_t cmp_data[];
} __attribute__((packed));
compile_time_assert(sizeof(struct non_imagette_header) == SPECIFIC_NON_IMAGETTE_HEADER_SIZE, NON_IMAGETTE_HEADER_T_SIZE_IS_NOT_CORRECT);
__extension__
struct cmp_entity {
- uint16_t asw_version_id; /* ICU ASW Version ID */
+ uint32_t version_id; /* ICU ASW/cmp_tool Version ID */
uint32_t cmp_ent_size:24; /* Compression Entity Size */
uint32_t original_size:24; /* Original Data Size */
union {
@@ -161,7 +163,7 @@ size_t cmp_ent_create(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
/* create a compression entity and set the header fields */
size_t cmp_ent_build(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
- uint16_t asw_version_id, uint64_t start_time,
+ uint32_t version_id, uint64_t start_time,
uint64_t end_time, uint16_t model_id, uint8_t model_counter,
struct cmp_info *info, struct cmp_cfg *cfg);
@@ -171,7 +173,7 @@ int cmp_ent_read_imagette_header(struct cmp_entity *ent, struct cmp_info *info);
/* set functions for generic compression entity header */
-int cmp_ent_set_asw_version_id(struct cmp_entity *ent, uint32_t asw_version_id);
+int cmp_ent_set_version_id(struct cmp_entity *ent, uint32_t version_id);
int cmp_ent_set_size(struct cmp_entity *ent, uint32_t cmp_ent_size);
int cmp_ent_set_original_size(struct cmp_entity *ent, uint32_t original_size);
@@ -186,7 +188,7 @@ int cmp_ent_set_fine_end_time(struct cmp_entity *ent, uint16_t fine_time);
int cmp_ent_set_data_type(struct cmp_entity *ent,
enum cmp_ent_data_type data_type);
-int cmp_ent_set_data_type_raw_bit(struct cmp_entity *ent, int raw_bit);
+int cmp_ent_data_type_valid(enum cmp_ent_data_type data_type);
int cmp_ent_set_cmp_mode(struct cmp_entity *ent, uint32_t cmp_mode_used);
int cmp_ent_set_model_value(struct cmp_entity *ent, uint32_t model_value_used);
int cmp_ent_set_model_id(struct cmp_entity *ent, uint32_t model_id);
@@ -233,7 +235,7 @@ int cmp_ent_set_non_ima_cmp_par6(struct cmp_entity *ent, uint32_t cmp_par6_used)
/* get functions for generic compression entity header */
-uint16_t cmp_ent_get_asw_version_id(struct cmp_entity *ent);
+uint32_t cmp_ent_get_version_id(struct cmp_entity *ent);
uint32_t cmp_ent_get_size(struct cmp_entity *ent);
uint32_t cmp_ent_get_original_size(struct cmp_entity *ent);
diff --git a/include/cmp_io.h b/include/cmp_io.h
index 1ad04e5d6b0263fb637da904fb25bc2005d849e8..77dce35b4f571b485267a8fae2ed74b726c38f2b 100644
--- a/include/cmp_io.h
+++ b/include/cmp_io.h
@@ -41,7 +41,7 @@ ssize_t read_file32(const char *file_name, uint32_t *buf, uint32_t samples,
ssize_t read_file_cmp_entity(const char *file_name, struct cmp_entity *ent,
uint32_t ent_size, int verbose_en);
-uint16_t cmp_tool_gen_version_id(const char *version);
+uint32_t cmp_tool_gen_version_id(const char *version);
int write_cmp_data_file(const void *buf, uint32_t buf_size, const char
*output_prefix, const char *name_extension, int verbose);
diff --git a/lib/cmp_entity.c b/lib/cmp_entity.c
index 5572f293752ed09f642a9f1bc98865775692920a..84f784d0993f62f12336966ce398ec7ff33f3e1d 100644
--- a/lib/cmp_entity.c
+++ b/lib/cmp_entity.c
@@ -84,6 +84,9 @@ uint32_t cmp_ent_cal_hdr_size(enum cmp_ent_data_type data_type)
return 0;
}
+ if ((data_type >> RAW_BIT_DATA_TYPE_POS) & 1U)
+ return GENERIC_HEADER_SIZE;
+
return 0;
}
@@ -109,20 +112,17 @@ int cmp_ent_data_type_valid(enum cmp_ent_data_type data_type)
* @brief set ICU ASW Version ID in the compression entity header
*
* @param ent pointer to a compression entity
- * @param asw_version_id the applications software version identifier
+ * @param version_id the applications software version identifier
*
* @returns 0 on success, otherwise error
*/
-int cmp_ent_set_asw_version_id(struct cmp_entity *ent, uint32_t asw_version_id)
+int cmp_ent_set_version_id(struct cmp_entity *ent, uint32_t version_id)
{
if (!ent)
return -1;
- if (asw_version_id > UINT16_MAX)
- return -1;
-
- ent->asw_version_id = cpu_to_be16(asw_version_id);
+ ent->version_id = cpu_to_be32(version_id);
return 0;
}
@@ -326,7 +326,8 @@ int cmp_ent_set_fine_end_time(struct cmp_entity *ent, uint16_t fine_time)
/**
* @brief set the data product type in the compression entity header
*
- * @param ent pointer to a compression entity
+ * @param ent pointer to a compression entity (including the
+ * uncompressed data bit)
* @param data_type compression entity data product type
*
* @returns 0 on success, otherwise error
@@ -338,44 +339,15 @@ int cmp_ent_set_data_type(struct cmp_entity *ent,
if (!ent)
return -1;
- if ((unsigned int)data_type > 0x7FFF)
+ if (data_type > UINT16_MAX)
return -1;
- data_type |= cmp_ent_get_data_type_raw_bit(ent) << RAW_BIT_IN_DATA_TYPE;
-
ent->data_type = cpu_to_be16(data_type);
return 0;
}
-/**
- * @brief set the raw bit in the data product field of the compression entity header
- *
- * @param ent pointer to a compression entity
- * @param raw_bit raw bit is set if raw_bit is non zero
- *
- * @returns 0 on success, otherwise error
- */
-
-int cmp_ent_set_data_type_raw_bit(struct cmp_entity *ent, int raw_bit)
-{
- uint16_t data_type;
-
- if (!ent)
- return -1;
-
- if (raw_bit)
- data_type = cpu_to_be16(ent->data_type) | 1UL << RAW_BIT_IN_DATA_TYPE;
- else
- data_type = cpu_to_be16(ent->data_type) & ~(1UL << RAW_BIT_IN_DATA_TYPE);
-
- ent->data_type = cpu_to_be16(data_type);
-
- return 0;
-}
-
-
/**
* @brief set the used compression mode in the compression entity header
*
@@ -967,12 +939,12 @@ int cmp_ent_set_non_ima_cmp_par6(struct cmp_entity *ent, uint32_t cmp_par_6_used
* @returns the ASW version identifier on success, 0 on error
*/
-uint16_t cmp_ent_get_asw_version_id(struct cmp_entity *ent)
+uint32_t cmp_ent_get_version_id(struct cmp_entity *ent)
{
if (!ent)
return 0;
- return be16_to_cpu(ent->asw_version_id);
+ return be32_to_cpu(ent->version_id);
}
@@ -1133,7 +1105,8 @@ uint16_t cmp_ent_get_fine_end_time(struct cmp_entity *ent)
*
* @param ent pointer to a compression entity
*
- * @returns the data_type on success, DATA_TYPE_UNKOWN on error
+ * @returns the data_type (including the uncompressed data bit) on success,
+ * DATA_TYPE_UNKOWN on error
*/
enum cmp_ent_data_type cmp_ent_get_data_type(struct cmp_entity *ent)
@@ -1141,7 +1114,7 @@ enum cmp_ent_data_type cmp_ent_get_data_type(struct cmp_entity *ent)
if (!ent)
return DATA_TYPE_UNKOWN;
- enum cmp_ent_data_type data_type = be16_to_cpu(ent->data_type) & 0X7FFF;
+ enum cmp_ent_data_type data_type = be16_to_cpu(ent->data_type);
if (cmp_ent_data_type_valid(data_type))
return data_type;
@@ -1163,7 +1136,7 @@ int cmp_ent_get_data_type_raw_bit(struct cmp_entity *ent)
if (!ent)
return 0;
- return (be16_to_cpu(ent->data_type) >> RAW_BIT_IN_DATA_TYPE) & 1U;
+ return (be16_to_cpu(ent->data_type) >> RAW_BIT_DATA_TYPE_POS) & 1U;
}
@@ -1620,6 +1593,9 @@ void *cmp_ent_get_data_buf(struct cmp_entity *ent)
if (!ent)
return NULL;
+ if (cmp_ent_get_data_type_raw_bit(ent))
+ return (uint8_t *)ent + GENERIC_HEADER_SIZE;
+
data_type = cmp_ent_get_data_type(ent);
switch (data_type) {
@@ -1839,13 +1815,15 @@ static int cmp_ent_write_cmp_pars(struct cmp_entity *ent, struct cmp_info *info,
return -1;
if (cmp_ent_set_cmp_mode(ent, info->cmp_mode_used))
return -1;
- if (cmp_ent_set_data_type_raw_bit(ent, raw_mode_is_used(info->cmp_mode_used)))
- return -1;
if (cmp_ent_set_model_value(ent, info->model_value_used))
return -1;
if (cmp_ent_set_lossy_cmp_par(ent, info->round_used))
return -1;
+ if (cmp_ent_get_data_type_raw_bit(ent))
+ /* no specific header is used for raw data we are done */
+ return 0;
+
switch (cmp_ent_get_data_type(ent)) {
case DATA_TYPE_IMAGETTE_ADAPTIVE:
case DATA_TYPE_SAT_IMAGETTE_ADAPTIVE:
@@ -1956,7 +1934,7 @@ size_t cmp_ent_create(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
* @param ent pointer to a compression entity; if NULL, the
* function returns the needed size
* @param data_type compression entity data product type
- * @param asw_version_id applications software version identifier
+ * @param version_id applications software version identifier
* @param start_time compression start timestamp (coarse and fine)
* @param end_time compression end timestamp (coarse and fine)
* @param model_id model identifier
@@ -1969,7 +1947,7 @@ size_t cmp_ent_create(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
*/
size_t cmp_ent_build(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
- uint16_t asw_version_id, uint64_t start_time,
+ uint32_t version_id, uint64_t start_time,
uint64_t end_time, uint16_t model_id, uint8_t model_counter,
struct cmp_info *info, struct cmp_cfg *cfg)
{
@@ -1983,7 +1961,7 @@ size_t cmp_ent_build(struct cmp_entity *ent, enum cmp_ent_data_type data_type,
return 0;
if (ent) {
- if (cmp_ent_set_asw_version_id(ent, asw_version_id))
+ if (cmp_ent_set_version_id(ent, version_id))
return 0;
if (cmp_ent_set_start_timestamp(ent, start_time))
return 0;
@@ -2216,20 +2194,20 @@ void cmp_ent_print(struct cmp_entity *ent)
static void cmp_ent_parse_generic_header(struct cmp_entity *ent)
{
- uint32_t asw_version_id, cmp_ent_size, original_size, cmp_mode_used,
+ uint32_t version_id, cmp_ent_size, original_size, cmp_mode_used,
model_value_used, model_id, model_counter, lossy_cmp_par_used,
start_coarse_time, end_coarse_time;
uint16_t start_fine_time, end_fine_time;
enum cmp_ent_data_type data_type;
int raw_bit;
- asw_version_id = cmp_ent_get_asw_version_id(ent);
- if (asw_version_id & CMP_TOOL_VERSION_ID_BIT) {
- uint16_t major = (asw_version_id & 0x7F00U) >> 8U;
- uint16_t minor = asw_version_id & 0x00FFU;
- printf("Compressed with cmp_tool version: %u.%02u\n", major,minor);
+ version_id = cmp_ent_get_version_id(ent);
+ if (version_id & CMP_TOOL_VERSION_ID_BIT) {
+ uint16_t major = (version_id & 0x7FFF0000U) >> 16U;
+ uint16_t minor = version_id & 0xFFFFU;
+ printf("Compressed with cmp_tool version: %u.%02u\n", major, minor);
} else
- printf("ICU ASW Version ID: %u\n", asw_version_id);
+ printf("ICU ASW Version ID: %u\n", version_id);
cmp_ent_size = cmp_ent_get_size(ent);
printf("Compression Entity Size: %u byte\n", cmp_ent_size);
diff --git a/lib/cmp_io.c b/lib/cmp_io.c
index f47ff8d60c6e194ea3275f682725beb09bab3385..9f7fd7294ee4884cc7624deea3588b6cad403dbb 100644
--- a/lib/cmp_io.c
+++ b/lib/cmp_io.c
@@ -937,6 +937,7 @@ static const char *skip_space(const char *str)
static const char *skip_comment(const char *str)
{
char c = *str;
+
if (c == '#') {
do {
str++;
@@ -1040,14 +1041,15 @@ static ssize_t str2uint8_arr(const char *str, uint8_t *data, uint32_t n_word,
errno = 0;
if (!data)
- n_word =~0;
+ n_word = ~0;
if (!file_name)
file_name = "unknown file name";
- for (i=0; i < n_word; ) {
+ for (i = 0; i < n_word; ) {
uint8_t read_val;
unsigned char c = *nptr;
+
if (c == '\0') {
if (!data) /* finished counting the sample */
break;
@@ -1140,15 +1142,15 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver
char *file_cpy = NULL;
long file_size;
ssize_t size;
+ size_t ret_code;
if (!file_name)
abort();
errno = 0;
- fp = fopen(file_name, "r");
- if (fp == NULL) {
+ fp = fopen(file_name, "rb");
+ if (fp == NULL)
goto fail;
- }
/* Get the number of bytes */
if (fseek(fp, 0L, SEEK_END) != 0)
@@ -1172,8 +1174,8 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t n_word, int ver
}
/* copy all the text into the file_cpy buffer */
- size_t ret_code = fread(file_cpy, sizeof(char), file_size, fp);
- if(ret_code != (size_t)file_size) {
+ ret_code = fread(file_cpy, sizeof(char), file_size, fp);
+ if (ret_code != (size_t)file_size) {
if (feof(fp))
printf("%s: %s: Error: unexpected end of file.\n", PROGRAM_NAME, file_name);
goto fail;
@@ -1304,7 +1306,8 @@ ssize_t read_file_cmp_entity(const char *file_name, struct cmp_entity *ent,
if (ent) {
enum cmp_ent_data_type data_type = cmp_ent_get_data_type(ent);
- if (data_type == DATA_TYPE_UNKOWN) {
+
+ if (!cmp_ent_data_type_valid(data_type)) {
fprintf(stderr, "%s: %s: Error: Compression data type is not supported.\n",
PROGRAM_NAME, file_name);
return -1;
@@ -1332,11 +1335,11 @@ ssize_t read_file_cmp_entity(const char *file_name, struct cmp_entity *ent,
* @returns version_id for the compression header; 0 on error
*/
-uint16_t cmp_tool_gen_version_id(const char *version)
+uint32_t cmp_tool_gen_version_id(const char *version)
{
char *copy, *token;
unsigned int n;
- uint16_t version_id;
+ uint32_t version_id;
/*
* version_id bits: msb |xxxx xxxx | xxxx xxxx| lsb
@@ -1348,11 +1351,11 @@ uint16_t cmp_tool_gen_version_id(const char *version)
copy = strdup(version);
token = strtok(copy, ".");
n = atoi(token);
- if (n > UINT8_MAX) {
+ if (n > UINT16_MAX) {
free(copy);
return 0;
}
- version_id = ((uint16_t)n) << 8U;
+ version_id = n << 16U;
if (version_id & CMP_TOOL_VERSION_ID_BIT) {
free(copy);
return 0;
@@ -1361,7 +1364,7 @@ uint16_t cmp_tool_gen_version_id(const char *version)
token = strtok(NULL, ".");
n = atoi(token);
free(copy);
- if (n > UINT8_MAX)
+ if (n > UINT16_MAX)
return 0;
version_id |= n;
diff --git a/test/cmp_tool/cmp_tool_integration_test.py b/test/cmp_tool/cmp_tool_integration_test.py
index 78d2be3c67c9b3bead804dc1f28cb280f24ae08a..f5a022b3538c3e280383eb2af22368bc3201a701 100644
--- a/test/cmp_tool/cmp_tool_integration_test.py
+++ b/test/cmp_tool/cmp_tool_integration_test.py
@@ -13,6 +13,11 @@ EXIT_FAILURE = 1
EXIT_SUCCESS = 0
+GENERIC_HEADER_SIZE = 32
+IMAGETTE_HEADER_SIZE = GENERIC_HEADER_SIZE+4
+IMAGETTE_ADAPTIVE_HEADER_SIZE = GENERIC_HEADER_SIZE+12
+NON_IMAGETTE_HEADER_SIZE = GENERIC_HEADER_SIZE+32
+
def call_cmp_tool(args):
args = shlex.split(PATH_CMP_TOOL + " " + args)
@@ -57,9 +62,8 @@ def cuc_timestamp(now):
def read_in_cmp_header(compressed_string):
- GENERIC_HEADER_SIZE = 30
- header = { 'asw_version_id' : { 'value': -1, 'bits': 16 },
+ header = { 'asw_version_id' : { 'value': -1, 'bits': 32 },
'cmp_ent_size' : { 'value': -1, 'bits': 24 },
'original_size' : { 'value': -1, 'bits': 24 },
'start_time' : { 'value': -1, 'bits': 48 },
@@ -69,7 +73,7 @@ def read_in_cmp_header(compressed_string):
'model_value_used' : { 'value': -1, 'bits': 8 },
'model_id' : { 'value': -1, 'bits': 16 },
'model_counter' : { 'value': -1, 'bits': 8 },
- 'spare' : { 'value': 0, 'bits': 8 },
+ 'spare' : { 'value': -1, 'bits': 8 },
'lossy_cmp_par_used' : { 'value': -1, 'bits': 16 },
'spill_used' : { 'value': -1, 'bits': 16 },
'golomb_par_used' : { 'value': -1, 'bits': 8 },
@@ -77,6 +81,7 @@ def read_in_cmp_header(compressed_string):
'ap1_golomb_par' : { 'value': -1, 'bits': 8 },
'ap2_spill_used' : { 'value': -1, 'bits': 16 },
'ap2_golomb_par' : { 'value': -1, 'bits': 8 },
+ 'spare_ap_ima' : { 'value': -1, 'bits': 8 },
'spill_1_used' : { 'value': -1, 'bits': 24 },
'cmp_par_1_used' : { 'value': -1, 'bits': 16 },
'spill_2_used' : { 'value': -1, 'bits': 24 },
@@ -89,6 +94,7 @@ def read_in_cmp_header(compressed_string):
'cmp_par_5_used' : { 'value': -1, 'bits': 16 },
'spill_6_used' : { 'value': -1, 'bits': 24 },
'cmp_par_6_used' : { 'value': -1, 'bits': 16 },
+ 'spare_non_ima' : { 'value': -1, 'bits': 16 },
'compressed_data' : { 'value': -1, 'bits': -1 }}
l = 0
@@ -102,13 +108,13 @@ def read_in_cmp_header(compressed_string):
header[data_field]['value'] = int(compressed_string[l:l+byte_len], 16)
l += byte_len
# end of the GENERIC_HEADER
- if l >= GENERIC_HEADER_SIZE*2:
+ if l/2 >= GENERIC_HEADER_SIZE:
break
- data_type = header['data_type']['value'] & 0x7FFF
+ data_type = int(header['data_type']['value'])
# Imagette Headers
if data_type == 1 or data_type == 2:
- for data_field in list(header)[12:18]:
+ for data_field in list(header)[12:19]:
if header[data_field]['bits'] % 4 != 0:
raise Exception("only work with 4 bit aligned data fields")
byte_len = header[data_field]['bits']//4
@@ -116,31 +122,57 @@ def read_in_cmp_header(compressed_string):
l += byte_len
# skip adaptive stuff if non adaptive header
if data_field == 'golomb_par_used' and data_type == 1:
- # l +=2
+ l += 2 # spare bits
+ assert(l/2==IMAGETTE_HEADER_SIZE)
break
- l += 2 # spare bits
+ if data_type == 2:
+ assert(l/2== NON_IMAGETTE_HEADER_SIZE)
# Non-Imagette Headers
elif data_type < 24:
- for data_field in list(header)[18:30]:
+ for data_field in list(header)[19:31]:
if header[data_field]['bits'] % 4 != 0:
raise Exception("only work with 4 bit aligned data fields")
byte_len = header[data_field]['bits']//4
header[data_field]['value'] = int(compressed_string[l:l+byte_len], 16)
l += byte_len
+ assert(l/2 == NON_IMAGETTE_HEADER_SIZE)
else:
- raise Exception("data_type unknown")
+ if (data_type >> 15) == 0: # check if raw mode is set
+ raise Exception("data_type unknown")
header['compressed_data']['value'] = compressed_string[l::]
# version conversion fuu
version_id = header['asw_version_id']['value']
- if version_id & 0x8000:
- header['asw_version_id']['value'] = "%.2f" % (int(version_id & 0x7F00)/256. + int(version_id&0xFF)*0.01)
+ if version_id & 0x80000000:
+ header['asw_version_id']['value'] = "%.2f" % (int(version_id & 0x7FFF0000)//0x8000 + int(version_id&0xFFFF)*0.01)
return header
+def build_generic_header(asw_version_id, cmp_ent_size, original_size,
+ start_time, end_time, data_type, cmp_mode_used,
+ model_value_used, model_id, model_counter,
+ lossy_cmp_par_used):
+
+ generic_header = ("%08X " % asw_version_id + "%06X " % cmp_ent_size +
+ "%06X " % original_size + "%012X " % start_time +
+ "%012X " % end_time + "%04X " % data_type +
+ "%02X " % cmp_mode_used + "%02X " % model_value_used +
+ "%04X " % model_id + "%02X " % model_counter+ "00 " +
+ "%04X " % lossy_cmp_par_used)
+
+ assert(len(generic_header.replace(" ", ""))/2 == GENERIC_HEADER_SIZE)
+ return generic_header
+
+
+def build_imagette_header(spill_used, golomb_par_used):
+ ima_header = "%04X %02X 00 " % (spill_used, golomb_par_used)
+ assert(len(ima_header.replace(" ", ""))/2 == IMAGETTE_HEADER_SIZE - GENERIC_HEADER_SIZE)
+ return ima_header
+
+
#get version
returncode, stdout, stderr = call_cmp_tool("--version")
assert(returncode == EXIT_SUCCESS)
@@ -378,9 +410,10 @@ def test_compression_diff():
assert(info['cmp_size'] == '20')
assert(info['cmp_err'] == '0')
else:
+ # import pdb; pdb.set_trace()
header = read_in_cmp_header(f.read())
assert(header['asw_version_id']['value'] == VERSION)
- assert(header['cmp_ent_size']['value'] == 34+4)
+ assert(header['cmp_ent_size']['value'] == IMAGETTE_HEADER_SIZE+4)
assert(header['original_size']['value'] == 10)
# todo
assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
@@ -565,7 +598,7 @@ def test_raw_mode_compression():
else:
header = read_in_cmp_header(f.read())
assert(header['asw_version_id']['value'] == VERSION)
- assert(header['cmp_ent_size']['value'] == 34+12)
+ assert(header['cmp_ent_size']['value'] == GENERIC_HEADER_SIZE+12)
assert(header['original_size']['value'] == 10)
# todo
assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
@@ -647,11 +680,12 @@ def test_guess_option():
exp_out = ('', '2', '', '7.27')
elif sub_test == 'guess_RDCU_model':
exp_out = (
- 'Importing model file model.dat ... DONE\n', '2', '', '0.23')
+ 'Importing model file model.dat ... DONE\n', '2', '', str(round((5*2)/(IMAGETTE_ADAPTIVE_HEADER_SIZE + 4), 2)))
#cmp_size:15bit-> 4byte cmp_data + 40byte header -> 16bit*5/(44Byte*8) '5.33'
elif sub_test == 'guess_level_3':
exp_out = (
- '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%', '0.26') #11.43
+ '', '3', ' 0%... 6%... 13%... 19%... 25%... 32%... 38%... 44%... 50%... 57%... 64%... 72%... 80%... 88%... 94%... 100%',
+ str(round((5*2)/(IMAGETTE_HEADER_SIZE + 4), 2))) #11.43
# cmp_size:7 bit -> 4byte cmp_data + 34 byte header -> 16bit*5/(40Byte*8)
else:
exp_out = ('', '', '')
@@ -891,11 +925,35 @@ def test_wrong_formart_cmp_fiel():
def test_sample_used_is_to_big():
- cmp_data_header = '80 07 00 00 26 00 00 0E 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 02 08 42 23 13 00 00 00 00 3C 07 00 44 44 44 00 \n'
- # ^ wrong samples_used
- cmp_data_no_header = '44 44 44 00 \n'
+ cmp_data = '44444400'
+
+ version_id = 0x8001_0042
+ cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
+ original_size = 0xE # wrong original_size correct is 0xA
+
+ start_time = cuc_timestamp(datetime.utcnow())
+ end_time = cuc_timestamp(datetime.utcnow())
+
+ data_type = 1
+ cmp_mode_used = 2
+ model_value_used = 8
+ model_id = 0xBEEF
+ model_counter = 0
+ lossy_cmp_par_used = 0
+
+ generic_header = build_generic_header(version_id, cmp_ent_size,
+ original_size, start_time, end_time, data_type,
+ cmp_mode_used, model_value_used, model_id,
+ model_counter, lossy_cmp_par_used)
+ spill_used = 60
+ golomb_par_used = 7
+
+ ima_header = build_imagette_header(spill_used, golomb_par_used)
+ cmp_data_header = generic_header + ima_header + cmp_data
+
info = ("cmp_size = 20\n" + "golomb_par_used = 1\n" + "spill_used = 4\n"
+ "cmp_mode_used = 2\n" +"samples_used=5\n")
+ # ^ wrong samples_used
cmp_file_name = 'big_cmp_size.cmp'
info_file_name = 'big_cmp_size.info'
pars = ['-i %s -d %s --no_header' % (info_file_name, cmp_file_name),
@@ -908,7 +966,7 @@ def test_sample_used_is_to_big():
for par in pars:
with open(cmp_file_name, 'w') as f:
if '--no_header' in par:
- f.write(cmp_data_no_header)
+ f.write(cmp_data)
else:
f.write(cmp_data_header)
@@ -934,8 +992,9 @@ def test_sample_used_is_to_big():
del_file(info_file_name)
-def test_read_in_header():
- cmp_file_name = 'read_in_header.cmp'
+
+def test_header_wrong_formatted():
+ cmp_file_name = 'read_wrong_format_header.cmp'
# packet wrong formatted
header = '80 07 00 00 !'
@@ -948,33 +1007,77 @@ def test_read_in_header():
"Importing compressed data file %s ... FAILED\n" % (cmp_file_name))
assert(stderr == "cmp_tool: %s: Error read in '!'. The data are not correct formatted.\n" % (cmp_file_name))
- # packet to small
- header = '80 07 00 00 26 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 02 08 42 23 13 00 00 00 00 3C 07 00 44 \n'
- # packet cut-off ^^ ^^
+ finally:
+ del_file(cmp_file_name)
+
+def test_header_read_in():
+ cmp_file_name = 'test_header_read_in.cmp'
+
+ cmp_data = '44444400'
+
+ version_id = 0x8001_0042
+ cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
+ original_size = 0xA
+
+ start_time = cuc_timestamp(datetime.utcnow())
+ end_time = cuc_timestamp(datetime.utcnow())
+
+ data_type = 1
+ cmp_mode_used = 2
+ model_value_used = 8
+ model_id = 0xBEEF
+ model_counter = 0
+ lossy_cmp_par_used = 0
+
+ generic_header = build_generic_header(version_id, cmp_ent_size,
+ original_size, start_time, end_time, data_type,
+ cmp_mode_used, model_value_used, model_id,
+ model_counter, lossy_cmp_par_used)
+ spill_used = 60
+ golomb_par_used = 7
+
+ ima_header = build_imagette_header(spill_used, golomb_par_used)
+ cmp_ent = generic_header + ima_header + cmp_data
+ # packet to small
+ cmp_ent = cmp_ent[:-4]
+ try:
with open(cmp_file_name, 'w') as f:
- f.write(header)
+ f.write(cmp_ent)
returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name))
assert(returncode == EXIT_FAILURE)
assert(stdout == CMP_START_STR_DECMP +
"Importing compressed data file %s ... FAILED\n" % (cmp_file_name))
assert(stderr == "cmp_tool: %s: The size of the compression entity set in the header of the compression entity is not the same size as the read-in file has.\n" %(cmp_file_name))
- # packet false data type
- header = '80 07 00 00 26 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 7F FE 02 08 42 23 13 00 00 00 00 3C 07 00 44 44 44 00 \n'
- # ^^ ^^ false data type
+ # false data type
+ data_type = 0x7FFE
+ generic_header = build_generic_header(version_id, cmp_ent_size,
+ original_size, start_time, end_time, data_type,
+ cmp_mode_used, model_value_used, model_id,
+ model_counter, lossy_cmp_par_used)
+ ima_header = build_imagette_header(spill_used, golomb_par_used)
+ cmp_ent = generic_header + ima_header + cmp_data
+ data_type = 1
+
with open(cmp_file_name, 'w') as f:
- f.write(header)
+ f.write(cmp_ent)
returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name))
assert(returncode == EXIT_FAILURE)
assert(stdout == CMP_START_STR_DECMP +
"Importing compressed data file %s ... FAILED\n" % (cmp_file_name))
assert(stderr == "cmp_tool: %s: Error: Compression data type is not supported.\n" % (cmp_file_name))
- # packet false data type
- header = '80 07 00 00 26 00 00 0C 03 9E 1D 5A 67 76 03 9E 1D 5A 67 9F 00 01 FF 08 42 23 13 00 00 00 00 3C 07 00 44 44 44 00 \n'
- # ^^ false cmp_mode_used
+ # false cmp_mode_used
+ cmp_mode_used = 0xFF
+ generic_header = build_generic_header(version_id, cmp_ent_size,
+ original_size, start_time, end_time, data_type,
+ cmp_mode_used, model_value_used, model_id,
+ model_counter, lossy_cmp_par_used)
+ ima_header = build_imagette_header(spill_used, golomb_par_used)
+ cmp_ent = generic_header + ima_header + cmp_data
+ cmp_mode_used = 2
with open(cmp_file_name, 'w') as f:
- f.write(header)
+ f.write(cmp_ent)
returncode, stdout, stderr = call_cmp_tool('-d %s' % (cmp_file_name))
assert(returncode == EXIT_FAILURE)
assert(stdout == CMP_START_STR_DECMP +