diff --git a/lib/common/cmp_entity.c b/lib/common/cmp_entity.c
index 056b41fa49337c4ba758bf929bf0f28daebe621c..afece91b260287c95dcd85b8a47eb06329a04e78 100644
--- a/lib/common/cmp_entity.c
+++ b/lib/common/cmp_entity.c
@@ -2282,7 +2282,7 @@ static void cmp_ent_parse_generic_header(const struct cmp_entity *ent)
debug_print("Compressed with cmp_tool version: %u.%02u", major, minor);
} else
- debug_print("ICU ASW Version ID: %08" PRIx32, version_id);
+ debug_print("ICU ASW Version ID: 0x%08" PRIx32, version_id);
cmp_ent_size = cmp_ent_get_size(ent);
debug_print("Compression Entity Size: %" PRIu32 " byte", cmp_ent_size);
@@ -2291,16 +2291,16 @@ static void cmp_ent_parse_generic_header(const struct cmp_entity *ent)
debug_print("Original Data Size: %" PRIu32 " byte", original_size);
start_coarse_time = cmp_ent_get_coarse_start_time(ent);
- debug_print("Compression Coarse Start Time: %" PRIu32, start_coarse_time);
+ debug_print("Compression Coarse Start Time: 0x%" PRIx32, start_coarse_time);
start_fine_time = cmp_ent_get_fine_start_time(ent);
- debug_print("Compression Fine Start Time: %d", start_fine_time);
+ debug_print("Compression Fine Start Time: 0x%x", start_fine_time);
end_coarse_time = cmp_ent_get_coarse_end_time(ent);
- debug_print("Compression Coarse End Time: %" PRIu32, end_coarse_time);
+ debug_print("Compression Coarse End Time: 0x%" PRIx32, end_coarse_time);
end_fine_time = cmp_ent_get_fine_end_time(ent);
- debug_print("Compression Fine End Time: %d", end_fine_time);
+ debug_print("Compression Fine End Time: 0x%x", end_fine_time);
#ifdef HAS_TIME_H
{
diff --git a/meson.build b/meson.build
index 7f7954389c057dd32fe8f0b43755a7b254f69867..58c5545974f0d5eef62342dfa6ff1b2243647024 100644
--- a/meson.build
+++ b/meson.build
@@ -1,5 +1,5 @@
project('cmp_tool', 'c',
- version : '0.12-b4',
+ version : '0.12',
meson_version : '>= 0.63',
license : 'GPL-2.0',
default_options : [
diff --git a/programs/cmp_io.c b/programs/cmp_io.c
index 81931f19b9a294c7c8b43e90a159f747b4deaa72..9b026343debc55ac6db8093871cb0ceb173b7eee 100644
--- a/programs/cmp_io.c
+++ b/programs/cmp_io.c
@@ -1326,7 +1326,7 @@ ssize_t read_file8(const char *file_name, uint8_t *buf, uint32_t buf_size, int f
if (flags & CMP_IO_BINARY) {
if (buf) {
ret_code = fread(buf, sizeof(uint8_t), buf_size, fp);
- if (ret_code != (size_t)file_size) {
+ if (ret_code != (size_t)buf_size) {
if (feof(fp))
printf("%s: %s: Error: unexpected end of file.\n", PROGRAM_NAME, file_name);
goto fail;
diff --git a/programs/cmp_tool.c b/programs/cmp_tool.c
index aaa947d1acd65fa590e01117e6881fd0e33b5f50..fc608b2d888436891c0c97540e4e000c0a942419 100644
--- a/programs/cmp_tool.c
+++ b/programs/cmp_tool.c
@@ -39,7 +39,6 @@
#define BUFFER_LENGTH_DEF_FAKTOR 2
#define DEFAULT_MODEL_ID 53264 /* random default id */
-#define DEFAULT_MODEL_COUNTER 0
/* parse a data_type option argument */
@@ -117,11 +116,11 @@ static int io_flags;
/* if non zero add a compression entity header in front of the compressed data */
static int include_cmp_header = 1;
-/* model ID string set by the --model_id option */
-static const char *model_id_str;
+/* model ID set by the --model_id option */
+static uint32_t model_id = DEFAULT_MODEL_ID;
-/* model counter string set by the --model_counter option */
-static const char *model_counter_str;
+/* model counter set by the --model_counter option */
+static uint32_t model_counter;
/**
@@ -241,10 +240,20 @@ int main(int argc, char **argv)
include_cmp_header = 0;
break;
case MODEL_ID:
- model_id_str = optarg;
+ if (atoui32("model_id", optarg, &model_id))
+ return -1;
+ if (model_counter > UINT16_MAX) {
+ fprintf(stderr, "%s: Error: model id value to large.\n", PROGRAM_NAME);
+ return -1;
+ }
break;
case MODEL_COUTER:
- model_counter_str = optarg;
+ if (atoui32("model_counter", optarg, &model_counter))
+ return -1;
+ if (model_counter > UINT8_MAX) {
+ fprintf(stderr, "%s: Error: model counter value to large.\n", PROGRAM_NAME);
+ return -1;
+ }
break;
default:
print_help(program_name);
@@ -723,6 +732,15 @@ static int cmp_gernate_rdcu_info(const struct cmp_cfg *cfg, int cmp_size_bit,
}
+/**
+ * retrun a current PLATO timestamp
+ */
+uint64_t return_timestamp(void)
+{
+ return cmp_ent_create_timestamp(NULL);
+}
+
+
/**
* @brief compress chunk data and write the results to files
*/
@@ -732,9 +750,10 @@ static int compression_of_chunk(void *chunk, uint32_t size, void *model, struct
uint32_t bound = compress_chunk_cmp_size_bound(chunk, size);
uint32_t *cmp_data;
uint32_t cmp_size;
- enum cmp_error cmp_error;
int error;
+ compress_chunk_init(&return_timestamp, cmp_tool_gen_version_id(CMP_TOOL_VERSION));
+
if (!bound)
return -1;
cmp_data = calloc(1, bound);
@@ -746,20 +765,26 @@ static int compression_of_chunk(void *chunk, uint32_t size, void *model, struct
printf("Compress chunk data ... ");
cmp_size = compress_chunk(chunk, size, model, model,
cmp_data, bound, chunk_par);
- cmp_error = cmp_get_error_code(cmp_size);
- if (cmp_error != CMP_ERROR_NO_ERROR) {
- fprintf(stderr, "%s\n", cmp_get_error_string(cmp_error));
- free(cmp_data);
- cmp_data = NULL;
- printf("FAILED\n");
- return -1;
- }
+ if (cmp_is_error(cmp_size))
+ goto cmp_chunk_fail;
+
+ cmp_size = compress_chunk_set_model_id_and_counter(cmp_data, cmp_size,
+ (uint16_t)model_id, (uint8_t)model_counter);
+ if (cmp_is_error(cmp_size))
+ goto cmp_chunk_fail;
printf("DONE\nWrite compressed data to file %s.cmp ... ", output_prefix);
error = write_data_to_file(cmp_data, (uint32_t)cmp_size, output_prefix,
".cmp", io_flags);
+
+cmp_chunk_fail:
free(cmp_data);
cmp_data = NULL;
+ if (cmp_is_error(cmp_size)) {
+ fprintf(stderr, "%s: %s.\n", PROGRAM_NAME, cmp_get_error_name(cmp_size));
+ printf("FAILED\n");
+ return (int)cmp_get_error_code(cmp_size);
+ }
if (error) {
printf("FAILED\n");
return -1;
@@ -781,8 +806,6 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info)
size_t s;
uint64_t start_time = cmp_ent_create_timestamp(NULL);
struct cmp_entity *cmp_entity = NULL;
- uint8_t model_counter = DEFAULT_MODEL_COUNTER;
- uint16_t model_id = DEFAULT_MODEL_ID;
void *data_to_write_to_file;
if (cfg->buffer_length == 0) {
@@ -843,29 +866,12 @@ static int compression(struct cmp_cfg *cfg, struct cmp_info *info)
goto error_cleanup;
}
- if (model_id_str) {
- uint32_t red_val;
-
- error = atoui32("model_id", model_id_str, &red_val);
- if (error || red_val > UINT16_MAX)
- return -1;
- model_id = (uint16_t)red_val;
- }
- if (model_counter_str) {
- uint32_t red_val;
-
- error = atoui32("model_counter", model_counter_str, &red_val);
- if (error || red_val > UINT8_MAX)
- return -1;
- model_counter = (uint8_t)red_val;
- } else {
- if (model_mode_is_used(cfg->cmp_mode))
- model_counter = DEFAULT_MODEL_COUNTER + 1;
- }
+ if (!model_counter && model_mode_is_used(cfg->cmp_mode))
+ model_counter++;
s = cmp_ent_build(cmp_entity, cmp_tool_gen_version_id(CMP_TOOL_VERSION),
- start_time, cmp_ent_create_timestamp(NULL), model_id,
- model_counter, cfg, cmp_size);
+ start_time, cmp_ent_create_timestamp(NULL), (uint16_t)model_id,
+ (uint8_t)model_counter, cfg, cmp_size);
if (!s) {
fprintf(stderr, "%s: error occurred while creating the compression entity header.\n", PROGRAM_NAME);
goto error_cleanup;
diff --git a/test/cmp_tool/cmp_tool_integration_test.py b/test/cmp_tool/cmp_tool_integration_test.py
index 5cadbf231cc39ebe8f12c13d95b9a791df00c7cf..8eb95da31adb180ccd3bb3895f63e9ff90ee6b7c 100755
--- a/test/cmp_tool/cmp_tool_integration_test.py
+++ b/test/cmp_tool/cmp_tool_integration_test.py
@@ -7,9 +7,11 @@ import os
import math
import shutil
from pathlib import Path
+import hashlib
from datetime import datetime
from datetime import timedelta
+from datetime import timezone
EXIT_FAILURE = 1
@@ -78,7 +80,7 @@ def del_directory(directoryPath):
def cuc_timestamp(now):
- epoch = datetime(2020, 1, 1)
+ epoch = datetime(2020, 1, 1, tzinfo=timezone.utc)
timestamp = (now - epoch).total_seconds()
cuc_coarse = int( math.floor(timestamp) * 256 * 256)
@@ -447,9 +449,9 @@ def test_compression_diff():
assert(header['cmp_ent_size']['value'] == IMAGETTE_HEADER_SIZE+3)
assert(header['original_size']['value'] == 10)
# todo
- assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['start_time']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
# todo
- assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
assert(header['data_type']['value'] == 1)
assert(header['cmp_mode_used']['value'] == 2)
# assert(header['model_value_used']['value'] == 8)
@@ -582,9 +584,9 @@ def test_model_compression():
assert(header['cmp_ent_size']['value'] == IMAGETTE_ADAPTIVE_HEADER_SIZE+2)
assert(header['original_size']['value'] == 10)
# todo
- assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['start_time']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
#todo
- assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
assert(header['data_type']['value'] == DATA_TYPE_IMAGETTE_ADAPTIVE)
assert(header['cmp_mode_used']['value'] == 3)
assert(header['model_value_used']['value'] == int(cfg['model_value']))
@@ -691,9 +693,9 @@ def test_raw_mode_compression():
assert(header['cmp_ent_size']['value'] == GENERIC_HEADER_SIZE+10)
assert(header['original_size']['value'] == 10)
# todo
- assert(header['start_time']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['start_time']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
#todo
- assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.utcnow()))
+ assert(header['end_timestamp']['value'] < cuc_timestamp(datetime.now(timezone.utc)))
assert(header['data_type']['value'] == 1+0x8000)
# assert(header['cmp_mode_used']['value'] == 2)
# assert(header['model_value_used']['value'] == 8)
@@ -1017,8 +1019,8 @@ def test_sample_used_is_to_big():
cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
original_size = 0xE # wrong original_size correct is 0xA
- start_time = cuc_timestamp(datetime.utcnow())
- end_time = cuc_timestamp(datetime.utcnow())
+ start_time = cuc_timestamp(datetime.now(timezone.utc))
+ end_time = cuc_timestamp(datetime.now(timezone.utc))
data_type = 1
cmp_mode_used = 2
@@ -1083,8 +1085,8 @@ def test_cmp_entity_not_4_byte_aligned():
cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
original_size = 0xC
- start_time = cuc_timestamp(datetime.utcnow())
- end_time = cuc_timestamp(datetime.utcnow())
+ start_time = cuc_timestamp(datetime.now(timezone.utc))
+ end_time = cuc_timestamp(datetime.now(timezone.utc))
data_type = 1
cmp_mode_used = 2
@@ -1177,8 +1179,8 @@ def test_header_read_in():
cmp_ent_size = IMAGETTE_HEADER_SIZE + len(cmp_data)//2
original_size = 0xA
- start_time = cuc_timestamp(datetime.utcnow())
- end_time = cuc_timestamp(datetime.utcnow())
+ start_time = cuc_timestamp(datetime.now(timezone.utc))
+ end_time = cuc_timestamp(datetime.now(timezone.utc))
data_type = 1
cmp_mode_used = 2
@@ -1414,7 +1416,6 @@ def test_rdcu_pkt():
assert(f1.read() == f2.read())
finally:
- pass
del_directory('TC_FILES')
del_file(data_file_name)
del_file(cfg_file_name)
@@ -1426,5 +1427,59 @@ def test_rdcu_pkt():
del_file(output_prefix2+'_upmodel.dat')
+def test_chunk_compression():
+ # decompress the test data
+ output_prefix1 = "ref_short_cadence_1"
+ cmp_data_path1 = "test/cmp_tool/ref_short_cadence_1_cmp.cmp"
+ output_prefix2 = "ref_short_cadence_2"
+ cmp_data_path2 = "test/cmp_tool/ref_short_cadence_2_cmp.cmp"
+
+ try:
+ returncode, stdout, stderr = call_cmp_tool(
+ "--binary -d " + cmp_data_path1 + " -o " + output_prefix1)
+
+ assert(stderr == "")
+ assert(stdout == CMP_START_STR_DECMP +
+ "Importing compressed data file %s ... DONE\n" % (cmp_data_path1) +
+ "Decompress data ... DONE\n" +
+ "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix1))
+ assert(returncode == EXIT_SUCCESS)
+
+ with open(output_prefix1 + '.dat', 'rb') as f:
+ sha1 = hashlib.sha1()
+ while True:
+ chunk = f.read(16 * 1024)
+ if not chunk:
+ break
+ sha1.update(chunk)
+
+ assert(sha1.hexdigest() == "7d8d94d2ac904f9ff4f934bb691b469a7391ce9e")
+
+ returncode, stdout, stderr = call_cmp_tool(
+ "--binary -d " + cmp_data_path2 + " -m " + output_prefix1 + ".dat -o " + output_prefix2)
+
+ assert(stderr == "")
+ assert(stdout == CMP_START_STR_DECMP +
+ "Importing compressed data file %s ... DONE\n" % (cmp_data_path2) +
+ "Importing model file %s.dat ... DONE\n" % (output_prefix1) +
+ "Decompress data ... DONE\n" +
+ "Write decompressed data to file %s.dat ... DONE\n" % (output_prefix2) +
+ "Write updated model to file %s_upmodel.dat ... DONE\n" % (output_prefix2))
+ assert(returncode == EXIT_SUCCESS)
+
+ with open(output_prefix2 + '.dat', 'rb') as f:
+ sha1 = hashlib.sha1()
+ while True:
+ chunk = f.read(16 * 1024)
+ if not chunk:
+ break
+ sha1.update(chunk)
+
+ assert(sha1.hexdigest() == "28ecc82a0c44ae7a461c26112b00f65b9b54e66a")
+
+ finally:
+ del_file(output_prefix1+'.dat')
+ del_file(output_prefix2+'.dat')
+ del_file(output_prefix2+'_upmodel.dat')
# TODO: random test
diff --git a/test/cmp_tool/gen_test_data.c b/test/cmp_tool/gen_test_data.c
new file mode 100644
index 0000000000000000000000000000000000000000..05c1ddab03ad7067e7e9465b80bd78bf1f3691de
--- /dev/null
+++ b/test/cmp_tool/gen_test_data.c
@@ -0,0 +1,41 @@
+#include <stdio.h>
+#include <string.h>
+
+#include "../bench/ref_short_cadence_1_cmp.h"
+#include "../bench/ref_short_cadence_2_cmp.h"
+
+
+int main(int argc, char *argv[])
+{
+ int i;
+ FILE *fp;
+ size_t s;
+
+ if (argc < 1)
+ return 1;
+
+ for (i = 1; i < argc; i++) {
+ if (strstr(argv[i], "ref_short_cadence_1_cmp")) {
+ fp = fopen(argv[i], "wb");
+ if(!fp)
+ return 1;
+ s = fwrite(ref_short_cadence_1_cmp, 1, ref_short_cadence_1_cmp_len, fp);
+ fclose(fp);
+ if (s!=ref_short_cadence_1_cmp_len)
+ return 1;
+ } else if (strstr(argv[i], "ref_short_cadence_2_cmp")) {
+ fp = fopen(argv[i], "wb");
+ if(!fp)
+ return 1;
+ s = fwrite(ref_short_cadence_2_cmp, 1, ref_short_cadence_2_cmp_len, fp);
+ fclose(fp);
+ if (s!=ref_short_cadence_2_cmp_len)
+ return 1;
+ } else {
+ fprintf(stderr,"Unknown test data\n");
+ return 1;
+ }
+ }
+
+ return 0;
+}
diff --git a/test/cmp_tool/meson.build b/test/cmp_tool/meson.build
index dfa0ce6d35ecffc3639fbe45c2b3a3a85b9d4198..907a7ee0da6a60fcc4537c2545bf691b304faafe 100644
--- a/test/cmp_tool/meson.build
+++ b/test/cmp_tool/meson.build
@@ -1,12 +1,22 @@
int_test_file = files('cmp_tool_integration_test.py')
+gen_test_data = executable('gen_test_data',
+ 'gen_test_data.c',
+ c_args : '-Wno-overlength-strings',
+)
+
+test_data = custom_target('gen_test_data',
+ output : ['ref_short_cadence_1_cmp.cmp', 'ref_short_cadence_2_cmp.cmp'],
+ command : [gen_test_data, '@OUTPUT@']
+)
+
pytest = find_program('pytest', required : false)
if pytest.found()
test('cmp_tool Interface Test',
pytest,
args : ['--color=yes', '-vvv', int_test_file],
env: test_env,
- depends : cmp_tool_exe,
+ depends : [cmp_tool_exe, test_data],
timeout : 100,
workdir : meson.project_build_root())
else